2026-02-22T21:31:13.489 INFO:root:teuthology version: 1.2.4.dev9+gfa17720d0 2026-02-22T21:31:13.517 DEBUG:teuthology.report:Pushing job info to https://paddles-paddles.apps.pok.os.sepia.ceph.com 2026-02-22T21:31:13.613 INFO:teuthology.run:Config: archive_path: /home/teuthworker/mnt/teuthology/yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial/64060 branch: wip-yuri6-testing-2026-02-22-1629-squid description: rbd/mirror-thrash/{base/install clients/mirror cluster/{2-node openstack} conf/{disable-pool-app} msgr-failures/few objectstore/bluestore-comp-lz4 policy/simple rbd-mirror/four-per-cluster supported-random-distro$/{centos_latest} workloads/rbd-mirror-snapshot-stress-workunit-exclusive-lock} email: yweinste@redhat.com first_in_suite: false flavor: default job_id: '64060' kernel: branch: distro kdb: true sha1: distro last_in_suite: false machine_type: trial meta: - desc: run two ceph clusters and install rbd-mirror - desc: configure the permissions for client.mirror - desc: 2 ceph clusters with 1 mon and 3 osds each - desc: run four rbd-mirror daemons per cluster - desc: run the rbd_mirror_stress.sh workunit to test the rbd-mirror daemon name: yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial no_nested_subset: false openstack: - volumes: count: 3 size: 30 os_type: centos os_version: 9.stream overrides: admin_socket: branch: wip-yuri6-testing-2026-02-22-1629-squid ceph: conf: client: debug rbd: 20 debug rbd_mirror: 15 log to stderr: false rbd default features: 125 rbd mirror image policy type: simple client.mirror.0: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid client.mirror.1: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid client.mirror.2: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid client.mirror.3: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid global: mon client directed command retry: 5 mon warn on pool no app: false ms inject socket failures: 5000 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: bluestore block size: 96636764160 bluestore compression algorithm: lz4 bluestore compression mode: aggressive bluestore fsck on mount: true debug bluefs: 1/20 debug bluestore: 1/20 debug ms: 1 debug osd: 20 debug rocksdb: 4/10 mon osd backfillfull_ratio: 0.85 mon osd full ratio: 0.9 mon osd nearfull ratio: 0.8 osd failsafe full ratio: 0.95 osd objectstore: bluestore flavor: default fs: xfs log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) - \(OSD_SLOW_PING_TIME sha1: f90f4eaebb28b619cf76e80cbbe997656da69579 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: f90f4eaebb28b619cf76e80cbbe997656da69579 selinux: allowlist: - scontext=system_u:system_r:getty_t:s0 thrashosds: bdev_inject_crash: 2 bdev_inject_crash_probability: 0.5 workunit: branch: wip-yuri6-testing-2026-02-22-1629-squid sha1: f90f4eaebb28b619cf76e80cbbe997656da69579 owner: scheduled_yuriw@soko04.front.sepia.ceph.com priority: 69 repo: https://github.com/ceph/ceph-ci.git roles: - - cluster1.mon.a - cluster1.mgr.x - cluster2.mgr.x - cluster1.osd.0 - cluster1.osd.1 - cluster1.osd.2 - cluster1.client.0 - cluster2.client.0 - - cluster2.mon.a - cluster2.osd.0 - cluster2.osd.1 - cluster2.osd.2 - cluster1.client.mirror - cluster1.client.mirror.0 - cluster1.client.mirror.1 - cluster1.client.mirror.2 - cluster1.client.mirror.3 - cluster1.client.mirror.4 - cluster1.client.mirror.5 - cluster1.client.mirror.6 - cluster2.client.mirror - cluster2.client.mirror.0 - cluster2.client.mirror.1 - cluster2.client.mirror.2 - cluster2.client.mirror.3 - cluster2.client.mirror.4 - cluster2.client.mirror.5 - cluster2.client.mirror.6 seed: 9196 sha1: f90f4eaebb28b619cf76e80cbbe997656da69579 sleep_before_teardown: 0 subset: 111/120000 suite: rbd suite_branch: wip-yuri6-testing-2026-02-22-1629-squid suite_path: /home/teuthworker/src/github.com_ceph_ceph-c_f90f4eaebb28b619cf76e80cbbe997656da69579/qa suite_relpath: qa suite_repo: https://github.com/ceph/ceph-ci.git suite_sha1: f90f4eaebb28b619cf76e80cbbe997656da69579 targets: trial004.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBDkPjQYprX4CF30+jAbTd9L3HjPvsP5GsEYCy0m1NpRHUc7PIKylXMo7Pk3bw2gAMNlmx9b6E/JJW/eTIjrHUGg= trial026.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBM5AXTZqVQNBK8dSoTzQSzjoGT4p0yrUpjtrv9BGY3wKuaRfx3V9PHQ5dxCaBeyOqt+DwHWF93iRGCUbLiIW/wQ= tasks: - install: extra_packages: - rbd-mirror - ceph: cluster: cluster1 - ceph: cluster: cluster2 - exec: cluster1.client.mirror.0: - sudo ceph --cluster cluster1 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd' - sudo ceph --cluster cluster1 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster1 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster1 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster1 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd' - rbd-mirror: client: cluster1.client.mirror.0 thrash: true - rbd-mirror: client: cluster1.client.mirror.1 thrash: true - rbd-mirror: client: cluster1.client.mirror.2 thrash: true - rbd-mirror: client: cluster1.client.mirror.3 thrash: true - rbd-mirror: client: cluster2.client.mirror.0 thrash: true - rbd-mirror: client: cluster2.client.mirror.1 thrash: true - rbd-mirror: client: cluster2.client.mirror.2 thrash: true - rbd-mirror: client: cluster2.client.mirror.3 thrash: true - rbd-mirror-thrash: cluster: cluster1 - rbd-mirror-thrash: cluster: cluster2 - workunit: clients: cluster1.client.mirror: - rbd/rbd_mirror_stress.sh env: CEPH_ARGS: '' MIRROR_POOL_MODE: image RBD_IMAGE_FEATURES: layering,exclusive-lock RBD_MIRROR_INSTANCES: '4' RBD_MIRROR_MODE: snapshot RBD_MIRROR_USE_EXISTING_CLUSTER: '1' RBD_MIRROR_USE_RBD_MIRROR: '1' timeout: 6h teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: fa17720d0088c3ac28e473468bfc79eeaff5cd38 timestamp: 2026-02-22_19:35:38 tube: trial user: yuriw verbose: true worker_log: /home/teuthworker/mnt/teuthology/worker_logs/dispatcher.trial.3318653 2026-02-22T21:31:13.613 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/github.com_ceph_ceph-c_f90f4eaebb28b619cf76e80cbbe997656da69579/qa; will attempt to use it 2026-02-22T21:31:13.613 INFO:teuthology.run:Found tasks at /home/teuthworker/src/github.com_ceph_ceph-c_f90f4eaebb28b619cf76e80cbbe997656da69579/qa/tasks 2026-02-22T21:31:13.614 INFO:teuthology.run_tasks:Running task internal.check_packages... 2026-02-22T21:31:13.615 INFO:teuthology.task.internal:Checking packages... 2026-02-22T21:31:13.615 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash 'f90f4eaebb28b619cf76e80cbbe997656da69579' 2026-02-22T21:31:13.615 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2026-02-22T21:31:13.615 INFO:teuthology.packaging:ref: None 2026-02-22T21:31:13.616 INFO:teuthology.packaging:tag: None 2026-02-22T21:31:13.616 INFO:teuthology.packaging:branch: wip-yuri6-testing-2026-02-22-1629-squid 2026-02-22T21:31:13.616 INFO:teuthology.packaging:sha1: f90f4eaebb28b619cf76e80cbbe997656da69579 2026-02-22T21:31:13.616 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=wip-yuri6-testing-2026-02-22-1629-squid 2026-02-22T21:31:13.736 INFO:teuthology.task.internal:Found packages for ceph version 19.2.3-689.gf90f4eae 2026-02-22T21:31:13.737 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2026-02-22T21:31:13.751 INFO:teuthology.task.internal:no buildpackages task found 2026-02-22T21:31:13.751 INFO:teuthology.run_tasks:Running task internal.save_config... 2026-02-22T21:31:13.766 INFO:teuthology.task.internal:Saving configuration 2026-02-22T21:31:13.780 INFO:teuthology.run_tasks:Running task internal.check_lock... 2026-02-22T21:31:13.792 INFO:teuthology.task.internal.check_lock:Checking locks... 2026-02-22T21:31:13.850 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'trial004.front.sepia.ceph.com', 'description': '/home/teuthworker/mnt/teuthology/yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial/64060', 'up': True, 'machine_type': 'trial', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2026-02-22 21:27:14.166711', 'locked_by': 'scheduled_yuriw@soko04.front.sepia.ceph.com', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBDkPjQYprX4CF30+jAbTd9L3HjPvsP5GsEYCy0m1NpRHUc7PIKylXMo7Pk3bw2gAMNlmx9b6E/JJW/eTIjrHUGg='} 2026-02-22T21:31:13.907 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'trial026.front.sepia.ceph.com', 'description': '/home/teuthworker/mnt/teuthology/yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial/64060', 'up': True, 'machine_type': 'trial', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2026-02-22 21:27:14.166092', 'locked_by': 'scheduled_yuriw@soko04.front.sepia.ceph.com', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBM5AXTZqVQNBK8dSoTzQSzjoGT4p0yrUpjtrv9BGY3wKuaRfx3V9PHQ5dxCaBeyOqt+DwHWF93iRGCUbLiIW/wQ='} 2026-02-22T21:31:13.908 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2026-02-22T21:31:13.920 INFO:teuthology.task.internal:roles: ubuntu@trial004.front.sepia.ceph.com - ['cluster1.mon.a', 'cluster1.mgr.x', 'cluster2.mgr.x', 'cluster1.osd.0', 'cluster1.osd.1', 'cluster1.osd.2', 'cluster1.client.0', 'cluster2.client.0'] 2026-02-22T21:31:13.920 INFO:teuthology.task.internal:roles: ubuntu@trial026.front.sepia.ceph.com - ['cluster2.mon.a', 'cluster2.osd.0', 'cluster2.osd.1', 'cluster2.osd.2', 'cluster1.client.mirror', 'cluster1.client.mirror.0', 'cluster1.client.mirror.1', 'cluster1.client.mirror.2', 'cluster1.client.mirror.3', 'cluster1.client.mirror.4', 'cluster1.client.mirror.5', 'cluster1.client.mirror.6', 'cluster2.client.mirror', 'cluster2.client.mirror.0', 'cluster2.client.mirror.1', 'cluster2.client.mirror.2', 'cluster2.client.mirror.3', 'cluster2.client.mirror.4', 'cluster2.client.mirror.5', 'cluster2.client.mirror.6'] 2026-02-22T21:31:13.920 INFO:teuthology.run_tasks:Running task console_log... 2026-02-22T21:31:14.051 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x74c5b03faca0>, signals=[15]) 2026-02-22T21:31:14.052 INFO:teuthology.run_tasks:Running task internal.connect... 2026-02-22T21:31:14.066 INFO:teuthology.task.internal:Opening connections... 2026-02-22T21:31:14.066 DEBUG:teuthology.task.internal:connecting to ubuntu@trial004.front.sepia.ceph.com 2026-02-22T21:31:14.067 DEBUG:teuthology.orchestra.connection:{'hostname': 'trial004.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2026-02-22T21:31:14.130 DEBUG:teuthology.task.internal:connecting to ubuntu@trial026.front.sepia.ceph.com 2026-02-22T21:31:14.131 DEBUG:teuthology.orchestra.connection:{'hostname': 'trial026.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2026-02-22T21:31:14.193 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2026-02-22T21:31:14.205 DEBUG:teuthology.orchestra.run.trial004:> uname -m 2026-02-22T21:31:14.221 INFO:teuthology.orchestra.run.trial004.stdout:x86_64 2026-02-22T21:31:14.221 DEBUG:teuthology.orchestra.run.trial004:> cat /etc/os-release 2026-02-22T21:31:14.277 INFO:teuthology.orchestra.run.trial004.stdout:NAME="CentOS Stream" 2026-02-22T21:31:14.277 INFO:teuthology.orchestra.run.trial004.stdout:VERSION="9" 2026-02-22T21:31:14.277 INFO:teuthology.orchestra.run.trial004.stdout:ID="centos" 2026-02-22T21:31:14.277 INFO:teuthology.orchestra.run.trial004.stdout:ID_LIKE="rhel fedora" 2026-02-22T21:31:14.277 INFO:teuthology.orchestra.run.trial004.stdout:VERSION_ID="9" 2026-02-22T21:31:14.277 INFO:teuthology.orchestra.run.trial004.stdout:PLATFORM_ID="platform:el9" 2026-02-22T21:31:14.277 INFO:teuthology.orchestra.run.trial004.stdout:PRETTY_NAME="CentOS Stream 9" 2026-02-22T21:31:14.277 INFO:teuthology.orchestra.run.trial004.stdout:ANSI_COLOR="0;31" 2026-02-22T21:31:14.277 INFO:teuthology.orchestra.run.trial004.stdout:LOGO="fedora-logo-icon" 2026-02-22T21:31:14.277 INFO:teuthology.orchestra.run.trial004.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2026-02-22T21:31:14.277 INFO:teuthology.orchestra.run.trial004.stdout:HOME_URL="https://centos.org/" 2026-02-22T21:31:14.277 INFO:teuthology.orchestra.run.trial004.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2026-02-22T21:31:14.277 INFO:teuthology.orchestra.run.trial004.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2026-02-22T21:31:14.277 INFO:teuthology.orchestra.run.trial004.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2026-02-22T21:31:14.278 INFO:teuthology.lock.ops:Updating trial004.front.sepia.ceph.com on lock server 2026-02-22T21:31:14.347 DEBUG:teuthology.orchestra.run.trial026:> uname -m 2026-02-22T21:31:14.365 INFO:teuthology.orchestra.run.trial026.stdout:x86_64 2026-02-22T21:31:14.365 DEBUG:teuthology.orchestra.run.trial026:> cat /etc/os-release 2026-02-22T21:31:14.422 INFO:teuthology.orchestra.run.trial026.stdout:NAME="CentOS Stream" 2026-02-22T21:31:14.422 INFO:teuthology.orchestra.run.trial026.stdout:VERSION="9" 2026-02-22T21:31:14.422 INFO:teuthology.orchestra.run.trial026.stdout:ID="centos" 2026-02-22T21:31:14.422 INFO:teuthology.orchestra.run.trial026.stdout:ID_LIKE="rhel fedora" 2026-02-22T21:31:14.422 INFO:teuthology.orchestra.run.trial026.stdout:VERSION_ID="9" 2026-02-22T21:31:14.422 INFO:teuthology.orchestra.run.trial026.stdout:PLATFORM_ID="platform:el9" 2026-02-22T21:31:14.422 INFO:teuthology.orchestra.run.trial026.stdout:PRETTY_NAME="CentOS Stream 9" 2026-02-22T21:31:14.422 INFO:teuthology.orchestra.run.trial026.stdout:ANSI_COLOR="0;31" 2026-02-22T21:31:14.422 INFO:teuthology.orchestra.run.trial026.stdout:LOGO="fedora-logo-icon" 2026-02-22T21:31:14.422 INFO:teuthology.orchestra.run.trial026.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2026-02-22T21:31:14.422 INFO:teuthology.orchestra.run.trial026.stdout:HOME_URL="https://centos.org/" 2026-02-22T21:31:14.422 INFO:teuthology.orchestra.run.trial026.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2026-02-22T21:31:14.423 INFO:teuthology.orchestra.run.trial026.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2026-02-22T21:31:14.423 INFO:teuthology.orchestra.run.trial026.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2026-02-22T21:31:14.423 INFO:teuthology.lock.ops:Updating trial026.front.sepia.ceph.com on lock server 2026-02-22T21:31:14.485 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2026-02-22T21:31:14.498 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2026-02-22T21:31:14.509 INFO:teuthology.task.internal:Checking for old test directory... 2026-02-22T21:31:14.509 DEBUG:teuthology.orchestra.run.trial004:> test '!' -e /home/ubuntu/cephtest 2026-02-22T21:31:14.511 DEBUG:teuthology.orchestra.run.trial026:> test '!' -e /home/ubuntu/cephtest 2026-02-22T21:31:14.526 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2026-02-22T21:31:14.537 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2026-02-22T21:31:14.537 DEBUG:teuthology.orchestra.run.trial004:> test -z $(ls -A /var/lib/ceph) 2026-02-22T21:31:14.566 DEBUG:teuthology.orchestra.run.trial026:> test -z $(ls -A /var/lib/ceph) 2026-02-22T21:31:14.582 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2026-02-22T21:31:14.747 INFO:teuthology.run_tasks:Running task kernel... 2026-02-22T21:31:14.767 INFO:teuthology.task.kernel:normalize config orig: {'branch': 'distro', 'kdb': True, 'sha1': 'distro'} 2026-02-22T21:31:14.767 INFO:teuthology.task.kernel:config {'cluster1.mon.a': {'branch': 'distro', 'kdb': True, 'sha1': 'distro'}, 'cluster2.mon.a': {'branch': 'distro', 'kdb': True, 'sha1': 'distro'}}, timeout 300 2026-02-22T21:31:14.767 DEBUG:teuthology.orchestra.run.trial004:> test -f /run/.containerenv -o -f /.dockerenv 2026-02-22T21:31:14.768 DEBUG:teuthology.orchestra.run.trial026:> test -f /run/.containerenv -o -f /.dockerenv 2026-02-22T21:31:14.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-22T21:31:14.782 DEBUG:teuthology.orchestra.run.trial004:> uname -r 2026-02-22T21:31:14.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-22T21:31:14.784 DEBUG:teuthology.orchestra.run.trial026:> uname -r 2026-02-22T21:31:14.838 INFO:teuthology.orchestra.run.trial004.stdout:5.14.0-681.el9.x86_64 2026-02-22T21:31:14.838 INFO:teuthology.task.kernel:Running kernel on trial004: 5.14.0-681.el9.x86_64 2026-02-22T21:31:14.839 DEBUG:teuthology.orchestra.run.trial004:> sudo yum install -y kernel 2026-02-22T21:31:14.841 INFO:teuthology.orchestra.run.trial026.stdout:5.14.0-681.el9.x86_64 2026-02-22T21:31:14.841 INFO:teuthology.task.kernel:Running kernel on trial026: 5.14.0-681.el9.x86_64 2026-02-22T21:31:14.841 DEBUG:teuthology.orchestra.run.trial026:> sudo yum install -y kernel 2026-02-22T21:31:16.263 INFO:teuthology.orchestra.run.trial004.stdout:CentOS Stream 9 - BaseOS 8.6 MB/s | 8.9 MB 00:01 2026-02-22T21:31:16.310 INFO:teuthology.orchestra.run.trial026.stdout:CentOS Stream 9 - BaseOS 8.2 MB/s | 8.9 MB 00:01 2026-02-22T21:31:19.406 INFO:teuthology.orchestra.run.trial026.stdout:CentOS Stream 9 - AppStream 10 MB/s | 27 MB 00:02 2026-02-22T21:31:19.697 INFO:teuthology.orchestra.run.trial004.stdout:CentOS Stream 9 - AppStream 9.2 MB/s | 27 MB 00:02 2026-02-22T21:31:23.247 INFO:teuthology.orchestra.run.trial004.stdout:CentOS Stream 9 - CRB 6.9 MB/s | 7.8 MB 00:01 2026-02-22T21:31:24.068 INFO:teuthology.orchestra.run.trial004.stdout:CentOS Stream 9 - Extras packages 193 kB/s | 20 kB 00:00 2026-02-22T21:31:24.618 INFO:teuthology.orchestra.run.trial004.stdout:Extra Packages for Enterprise Linux 42 MB/s | 20 MB 00:00 2026-02-22T21:31:27.748 INFO:teuthology.orchestra.run.trial004.stdout:lab-extras 2.9 MB/s | 50 kB 00:00 2026-02-22T21:31:28.763 INFO:teuthology.orchestra.run.trial004.stdout:Package kernel-5.14.0-645.el9.x86_64 is already installed. 2026-02-22T21:31:28.763 INFO:teuthology.orchestra.run.trial004.stdout:Package kernel-5.14.0-681.el9.x86_64 is already installed. 2026-02-22T21:31:28.783 INFO:teuthology.orchestra.run.trial004.stdout:Dependencies resolved. 2026-02-22T21:31:28.783 INFO:teuthology.orchestra.run.trial004.stdout:Nothing to do. 2026-02-22T21:31:28.784 INFO:teuthology.orchestra.run.trial004.stdout:Complete! 2026-02-22T21:31:28.807 DEBUG:teuthology.orchestra.run.trial004:> echo no | sudo yum reinstall kernel || true 2026-02-22T21:31:29.107 INFO:teuthology.orchestra.run.trial004.stdout:Last metadata expiration check: 0:00:02 ago on Sun 22 Feb 2026 09:31:27 PM UTC. 2026-02-22T21:31:29.187 INFO:teuthology.orchestra.run.trial004.stdout:Dependencies resolved. 2026-02-22T21:31:29.188 INFO:teuthology.orchestra.run.trial004.stdout:================================================================================ 2026-02-22T21:31:29.188 INFO:teuthology.orchestra.run.trial004.stdout: Package Architecture Version Repository Size 2026-02-22T21:31:29.188 INFO:teuthology.orchestra.run.trial004.stdout:================================================================================ 2026-02-22T21:31:29.189 INFO:teuthology.orchestra.run.trial004.stdout:Reinstalling: 2026-02-22T21:31:29.189 INFO:teuthology.orchestra.run.trial004.stdout: kernel x86_64 5.14.0-681.el9 baseos 803 k 2026-02-22T21:31:29.189 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:31:29.189 INFO:teuthology.orchestra.run.trial004.stdout:Transaction Summary 2026-02-22T21:31:29.189 INFO:teuthology.orchestra.run.trial004.stdout:================================================================================ 2026-02-22T21:31:29.189 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:31:29.189 INFO:teuthology.orchestra.run.trial004.stdout:Total download size: 803 k 2026-02-22T21:31:29.189 INFO:teuthology.orchestra.run.trial004.stdout:Installed size: 0 2026-02-22T21:31:29.190 INFO:teuthology.orchestra.run.trial004.stderr:Operation aborted. 2026-02-22T21:31:29.208 INFO:teuthology.orchestra.run.trial004.stdout:Is this ok [y/N]: 2026-02-22T21:31:29.209 DEBUG:teuthology.orchestra.run.trial004:> sudo yum reinstall -y kernel || true 2026-02-22T21:31:29.508 INFO:teuthology.orchestra.run.trial004.stdout:Last metadata expiration check: 0:00:02 ago on Sun 22 Feb 2026 09:31:27 PM UTC. 2026-02-22T21:31:29.588 INFO:teuthology.orchestra.run.trial004.stdout:Dependencies resolved. 2026-02-22T21:31:29.589 INFO:teuthology.orchestra.run.trial004.stdout:================================================================================ 2026-02-22T21:31:29.589 INFO:teuthology.orchestra.run.trial004.stdout: Package Architecture Version Repository Size 2026-02-22T21:31:29.589 INFO:teuthology.orchestra.run.trial004.stdout:================================================================================ 2026-02-22T21:31:29.589 INFO:teuthology.orchestra.run.trial004.stdout:Reinstalling: 2026-02-22T21:31:29.589 INFO:teuthology.orchestra.run.trial004.stdout: kernel x86_64 5.14.0-681.el9 baseos 803 k 2026-02-22T21:31:29.589 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:31:29.589 INFO:teuthology.orchestra.run.trial004.stdout:Transaction Summary 2026-02-22T21:31:29.590 INFO:teuthology.orchestra.run.trial004.stdout:================================================================================ 2026-02-22T21:31:29.590 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:31:29.590 INFO:teuthology.orchestra.run.trial004.stdout:Total download size: 803 k 2026-02-22T21:31:29.590 INFO:teuthology.orchestra.run.trial004.stdout:Installed size: 0 2026-02-22T21:31:29.590 INFO:teuthology.orchestra.run.trial004.stdout:Downloading Packages: 2026-02-22T21:31:30.393 INFO:teuthology.orchestra.run.trial026.stdout:CentOS Stream 9 - CRB 930 kB/s | 7.8 MB 00:08 2026-02-22T21:31:31.224 INFO:teuthology.orchestra.run.trial004.stdout:kernel-5.14.0-681.el9.x86_64.rpm 602 kB/s | 803 kB 00:01 2026-02-22T21:31:31.224 INFO:teuthology.orchestra.run.trial004.stdout:-------------------------------------------------------------------------------- 2026-02-22T21:31:31.224 INFO:teuthology.orchestra.run.trial004.stdout:Total 491 kB/s | 803 kB 00:01 2026-02-22T21:31:31.235 INFO:teuthology.orchestra.run.trial004.stdout:Running transaction check 2026-02-22T21:31:31.269 INFO:teuthology.orchestra.run.trial004.stdout:Transaction check succeeded. 2026-02-22T21:31:31.269 INFO:teuthology.orchestra.run.trial004.stdout:Running transaction test 2026-02-22T21:31:31.273 INFO:teuthology.orchestra.run.trial004.stdout:Transaction test succeeded. 2026-02-22T21:31:31.274 INFO:teuthology.orchestra.run.trial004.stdout:Running transaction 2026-02-22T21:31:31.297 INFO:teuthology.orchestra.run.trial004.stdout: Preparing : 1/1 2026-02-22T21:31:31.301 INFO:teuthology.orchestra.run.trial004.stdout: Reinstalling : kernel-5.14.0-681.el9.x86_64 1/2 2026-02-22T21:31:31.301 INFO:teuthology.orchestra.run.trial004.stdout: Cleanup : kernel-5.14.0-681.el9.x86_64 2/2 2026-02-22T21:31:31.413 INFO:teuthology.orchestra.run.trial004.stdout: Running scriptlet: kernel-5.14.0-681.el9.x86_64 2/2 2026-02-22T21:31:31.413 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : kernel-5.14.0-681.el9.x86_64 1/2 2026-02-22T21:31:31.467 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : kernel-5.14.0-681.el9.x86_64 2/2 2026-02-22T21:31:31.467 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:31:31.467 INFO:teuthology.orchestra.run.trial004.stdout:Reinstalled: 2026-02-22T21:31:31.468 INFO:teuthology.orchestra.run.trial004.stdout: kernel-5.14.0-681.el9.x86_64 2026-02-22T21:31:31.468 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:31:31.468 INFO:teuthology.orchestra.run.trial004.stdout:Complete! 2026-02-22T21:31:31.493 DEBUG:teuthology.orchestra.run.trial004:> rpm -q kernel | sort -rV | head -n 1 2026-02-22T21:31:31.537 INFO:teuthology.orchestra.run.trial026.stdout:CentOS Stream 9 - Extras packages 49 kB/s | 20 kB 00:00 2026-02-22T21:31:31.556 INFO:teuthology.orchestra.run.trial004.stdout:kernel-5.14.0-681.el9.x86_64 2026-02-22T21:31:31.558 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-681.el9.x86_64 2026-02-22T21:31:31.558 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2026-02-22T21:31:31.558 INFO:teuthology.task.kernel:Enabling kdb on cluster1.mon.a... 2026-02-22T21:31:31.558 DEBUG:teuthology.orchestra.run.trial004:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2026-02-22T21:31:31.621 INFO:teuthology.orchestra.run.trial004.stdout:ttyS1 2026-02-22T21:31:31.648 DEBUG:teuthology.parallel:result is None 2026-02-22T21:31:32.060 INFO:teuthology.orchestra.run.trial026.stdout:Extra Packages for Enterprise Linux 44 MB/s | 20 MB 00:00 2026-02-22T21:31:35.249 INFO:teuthology.orchestra.run.trial026.stdout:lab-extras 2.5 MB/s | 50 kB 00:00 2026-02-22T21:31:36.273 INFO:teuthology.orchestra.run.trial026.stdout:Package kernel-5.14.0-645.el9.x86_64 is already installed. 2026-02-22T21:31:36.274 INFO:teuthology.orchestra.run.trial026.stdout:Package kernel-5.14.0-681.el9.x86_64 is already installed. 2026-02-22T21:31:36.293 INFO:teuthology.orchestra.run.trial026.stdout:Dependencies resolved. 2026-02-22T21:31:36.294 INFO:teuthology.orchestra.run.trial026.stdout:Nothing to do. 2026-02-22T21:31:36.294 INFO:teuthology.orchestra.run.trial026.stdout:Complete! 2026-02-22T21:31:36.315 DEBUG:teuthology.orchestra.run.trial026:> echo no | sudo yum reinstall kernel || true 2026-02-22T21:31:36.652 INFO:teuthology.orchestra.run.trial026.stdout:Last metadata expiration check: 0:00:01 ago on Sun 22 Feb 2026 09:31:35 PM UTC. 2026-02-22T21:31:36.733 INFO:teuthology.orchestra.run.trial026.stdout:Dependencies resolved. 2026-02-22T21:31:36.733 INFO:teuthology.orchestra.run.trial026.stdout:================================================================================ 2026-02-22T21:31:36.733 INFO:teuthology.orchestra.run.trial026.stdout: Package Architecture Version Repository Size 2026-02-22T21:31:36.733 INFO:teuthology.orchestra.run.trial026.stdout:================================================================================ 2026-02-22T21:31:36.733 INFO:teuthology.orchestra.run.trial026.stdout:Reinstalling: 2026-02-22T21:31:36.733 INFO:teuthology.orchestra.run.trial026.stdout: kernel x86_64 5.14.0-681.el9 baseos 803 k 2026-02-22T21:31:36.733 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:31:36.734 INFO:teuthology.orchestra.run.trial026.stdout:Transaction Summary 2026-02-22T21:31:36.734 INFO:teuthology.orchestra.run.trial026.stdout:================================================================================ 2026-02-22T21:31:36.734 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:31:36.734 INFO:teuthology.orchestra.run.trial026.stdout:Total download size: 803 k 2026-02-22T21:31:36.734 INFO:teuthology.orchestra.run.trial026.stdout:Installed size: 0 2026-02-22T21:31:36.734 INFO:teuthology.orchestra.run.trial026.stderr:Operation aborted. 2026-02-22T21:31:36.754 INFO:teuthology.orchestra.run.trial026.stdout:Is this ok [y/N]: 2026-02-22T21:31:36.754 DEBUG:teuthology.orchestra.run.trial026:> sudo yum reinstall -y kernel || true 2026-02-22T21:31:37.061 INFO:teuthology.orchestra.run.trial026.stdout:Last metadata expiration check: 0:00:02 ago on Sun 22 Feb 2026 09:31:35 PM UTC. 2026-02-22T21:31:37.142 INFO:teuthology.orchestra.run.trial026.stdout:Dependencies resolved. 2026-02-22T21:31:37.142 INFO:teuthology.orchestra.run.trial026.stdout:================================================================================ 2026-02-22T21:31:37.142 INFO:teuthology.orchestra.run.trial026.stdout: Package Architecture Version Repository Size 2026-02-22T21:31:37.142 INFO:teuthology.orchestra.run.trial026.stdout:================================================================================ 2026-02-22T21:31:37.142 INFO:teuthology.orchestra.run.trial026.stdout:Reinstalling: 2026-02-22T21:31:37.142 INFO:teuthology.orchestra.run.trial026.stdout: kernel x86_64 5.14.0-681.el9 baseos 803 k 2026-02-22T21:31:37.143 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:31:37.143 INFO:teuthology.orchestra.run.trial026.stdout:Transaction Summary 2026-02-22T21:31:37.143 INFO:teuthology.orchestra.run.trial026.stdout:================================================================================ 2026-02-22T21:31:37.143 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:31:37.143 INFO:teuthology.orchestra.run.trial026.stdout:Total download size: 803 k 2026-02-22T21:31:37.143 INFO:teuthology.orchestra.run.trial026.stdout:Installed size: 0 2026-02-22T21:31:37.143 INFO:teuthology.orchestra.run.trial026.stdout:Downloading Packages: 2026-02-22T21:31:38.645 INFO:teuthology.orchestra.run.trial026.stdout:kernel-5.14.0-681.el9.x86_64.rpm 581 kB/s | 803 kB 00:01 2026-02-22T21:31:38.646 INFO:teuthology.orchestra.run.trial026.stdout:-------------------------------------------------------------------------------- 2026-02-22T21:31:38.646 INFO:teuthology.orchestra.run.trial026.stdout:Total 534 kB/s | 803 kB 00:01 2026-02-22T21:31:38.657 INFO:teuthology.orchestra.run.trial026.stdout:Running transaction check 2026-02-22T21:31:38.691 INFO:teuthology.orchestra.run.trial026.stdout:Transaction check succeeded. 2026-02-22T21:31:38.691 INFO:teuthology.orchestra.run.trial026.stdout:Running transaction test 2026-02-22T21:31:38.695 INFO:teuthology.orchestra.run.trial026.stdout:Transaction test succeeded. 2026-02-22T21:31:38.696 INFO:teuthology.orchestra.run.trial026.stdout:Running transaction 2026-02-22T21:31:38.719 INFO:teuthology.orchestra.run.trial026.stdout: Preparing : 1/1 2026-02-22T21:31:38.723 INFO:teuthology.orchestra.run.trial026.stdout: Reinstalling : kernel-5.14.0-681.el9.x86_64 1/2 2026-02-22T21:31:38.723 INFO:teuthology.orchestra.run.trial026.stdout: Cleanup : kernel-5.14.0-681.el9.x86_64 2/2 2026-02-22T21:31:38.835 INFO:teuthology.orchestra.run.trial026.stdout: Running scriptlet: kernel-5.14.0-681.el9.x86_64 2/2 2026-02-22T21:31:38.835 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : kernel-5.14.0-681.el9.x86_64 1/2 2026-02-22T21:31:38.888 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : kernel-5.14.0-681.el9.x86_64 2/2 2026-02-22T21:31:38.888 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:31:38.888 INFO:teuthology.orchestra.run.trial026.stdout:Reinstalled: 2026-02-22T21:31:38.888 INFO:teuthology.orchestra.run.trial026.stdout: kernel-5.14.0-681.el9.x86_64 2026-02-22T21:31:38.888 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:31:38.888 INFO:teuthology.orchestra.run.trial026.stdout:Complete! 2026-02-22T21:31:38.915 DEBUG:teuthology.orchestra.run.trial026:> rpm -q kernel | sort -rV | head -n 1 2026-02-22T21:31:38.939 INFO:teuthology.orchestra.run.trial026.stdout:kernel-5.14.0-681.el9.x86_64 2026-02-22T21:31:38.940 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-681.el9.x86_64 2026-02-22T21:31:38.940 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2026-02-22T21:31:38.940 INFO:teuthology.task.kernel:Enabling kdb on cluster2.mon.a... 2026-02-22T21:31:38.941 DEBUG:teuthology.orchestra.run.trial026:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2026-02-22T21:31:39.004 INFO:teuthology.orchestra.run.trial026.stdout:ttyS1 2026-02-22T21:31:39.027 DEBUG:teuthology.parallel:result is None 2026-02-22T21:31:39.028 INFO:teuthology.run_tasks:Running task internal.base... 2026-02-22T21:31:39.040 INFO:teuthology.task.internal:Creating test directory... 2026-02-22T21:31:39.040 DEBUG:teuthology.orchestra.run.trial004:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2026-02-22T21:31:39.043 DEBUG:teuthology.orchestra.run.trial026:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2026-02-22T21:31:39.084 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2026-02-22T21:31:39.095 INFO:teuthology.run_tasks:Running task internal.archive... 2026-02-22T21:31:39.108 INFO:teuthology.task.internal:Creating archive directory... 2026-02-22T21:31:39.109 DEBUG:teuthology.orchestra.run.trial004:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2026-02-22T21:31:39.111 DEBUG:teuthology.orchestra.run.trial026:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2026-02-22T21:31:39.143 INFO:teuthology.run_tasks:Running task internal.coredump... 2026-02-22T21:31:39.155 INFO:teuthology.task.internal:Enabling coredump saving... 2026-02-22T21:31:39.155 DEBUG:teuthology.orchestra.run.trial004:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2026-02-22T21:31:39.168 DEBUG:teuthology.orchestra.run.trial026:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2026-02-22T21:31:39.190 INFO:teuthology.orchestra.run.trial004.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2026-02-22T21:31:39.199 INFO:teuthology.orchestra.run.trial004.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2026-02-22T21:31:39.205 INFO:teuthology.orchestra.run.trial026.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2026-02-22T21:31:39.214 INFO:teuthology.orchestra.run.trial026.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2026-02-22T21:31:39.215 INFO:teuthology.run_tasks:Running task internal.sudo... 2026-02-22T21:31:39.228 INFO:teuthology.task.internal:Configuring sudo... 2026-02-22T21:31:39.228 DEBUG:teuthology.orchestra.run.trial004:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2026-02-22T21:31:39.242 DEBUG:teuthology.orchestra.run.trial026:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2026-02-22T21:31:39.279 INFO:teuthology.run_tasks:Running task internal.syslog... 2026-02-22T21:31:39.294 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2026-02-22T21:31:39.294 DEBUG:teuthology.orchestra.run.trial004:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2026-02-22T21:31:39.306 DEBUG:teuthology.orchestra.run.trial026:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2026-02-22T21:31:39.335 DEBUG:teuthology.orchestra.run.trial004:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2026-02-22T21:31:39.471 DEBUG:teuthology.orchestra.run.trial004:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2026-02-22T21:31:39.495 DEBUG:teuthology.orchestra.run.trial004:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2026-02-22T21:31:39.609 DEBUG:teuthology.orchestra.run.trial004:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2026-02-22T21:31:39.633 DEBUG:teuthology.orchestra.run.trial004:> set -ex 2026-02-22T21:31:39.633 DEBUG:teuthology.orchestra.run.trial004:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2026-02-22T21:31:39.701 DEBUG:teuthology.orchestra.run.trial026:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2026-02-22T21:31:39.811 DEBUG:teuthology.orchestra.run.trial026:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2026-02-22T21:31:39.836 DEBUG:teuthology.orchestra.run.trial026:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2026-02-22T21:31:39.991 DEBUG:teuthology.orchestra.run.trial026:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2026-02-22T21:31:40.016 DEBUG:teuthology.orchestra.run.trial026:> set -ex 2026-02-22T21:31:40.017 DEBUG:teuthology.orchestra.run.trial026:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2026-02-22T21:31:40.079 DEBUG:teuthology.orchestra.run.trial004:> sudo service rsyslog restart 2026-02-22T21:31:40.080 DEBUG:teuthology.orchestra.run.trial026:> sudo service rsyslog restart 2026-02-22T21:31:40.106 INFO:teuthology.orchestra.run.trial004.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2026-02-22T21:31:40.148 INFO:teuthology.orchestra.run.trial026.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2026-02-22T21:31:40.591 INFO:teuthology.run_tasks:Running task internal.timer... 2026-02-22T21:31:40.602 INFO:teuthology.task.internal:Starting timer... 2026-02-22T21:31:40.603 INFO:teuthology.run_tasks:Running task pcp... 2026-02-22T21:31:40.619 INFO:teuthology.run_tasks:Running task selinux... 2026-02-22T21:31:40.636 DEBUG:teuthology.task:Applying overrides for task selinux: {'allowlist': ['scontext=system_u:system_r:getty_t:s0']} 2026-02-22T21:31:40.637 DEBUG:teuthology.orchestra.run.trial004:> sudo service auditd rotate 2026-02-22T21:31:40.665 INFO:teuthology.orchestra.run.trial004.stdout:Rotating logs: 2026-02-22T21:31:40.667 DEBUG:teuthology.orchestra.run.trial026:> sudo service auditd rotate 2026-02-22T21:31:40.696 INFO:teuthology.orchestra.run.trial026.stdout:Rotating logs: 2026-02-22T21:31:40.698 DEBUG:teuthology.task.selinux:Getting current SELinux state 2026-02-22T21:31:40.699 DEBUG:teuthology.orchestra.run.trial004:> /usr/sbin/getenforce 2026-02-22T21:31:40.723 INFO:teuthology.orchestra.run.trial004.stdout:Permissive 2026-02-22T21:31:40.723 DEBUG:teuthology.orchestra.run.trial026:> /usr/sbin/getenforce 2026-02-22T21:31:40.755 INFO:teuthology.orchestra.run.trial026.stdout:Permissive 2026-02-22T21:31:40.755 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@trial004.front.sepia.ceph.com': 'permissive', 'ubuntu@trial026.front.sepia.ceph.com': 'permissive'} 2026-02-22T21:31:40.756 DEBUG:teuthology.orchestra.run.trial004:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:getty_t:s0 2026-02-22T21:31:40.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-22T21:31:40.789 DEBUG:teuthology.orchestra.run.trial026:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:getty_t:s0 2026-02-22T21:31:40.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-22T21:31:40.822 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2026-02-22T21:31:40.822 DEBUG:teuthology.orchestra.run.trial004:> sudo /usr/sbin/setenforce permissive 2026-02-22T21:31:40.852 DEBUG:teuthology.orchestra.run.trial026:> sudo /usr/sbin/setenforce permissive 2026-02-22T21:31:40.890 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2026-02-22T21:31:40.922 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2026-02-22T21:31:40.931 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2026-02-22T21:31:40.932 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit trial004.front.sepia.ceph.com,trial026.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2026-02-22T21:34:22.005 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@trial004.front.sepia.ceph.com'), Remote(name='ubuntu@trial026.front.sepia.ceph.com')] 2026-02-22T21:34:22.006 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@trial004.front.sepia.ceph.com' 2026-02-22T21:34:22.007 DEBUG:teuthology.orchestra.connection:{'hostname': 'trial004.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2026-02-22T21:34:22.117 DEBUG:teuthology.orchestra.run.trial004:> true 2026-02-22T21:34:22.192 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@trial004.front.sepia.ceph.com' 2026-02-22T21:34:22.193 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@trial026.front.sepia.ceph.com' 2026-02-22T21:34:22.193 DEBUG:teuthology.orchestra.connection:{'hostname': 'trial026.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2026-02-22T21:34:22.279 DEBUG:teuthology.orchestra.run.trial026:> true 2026-02-22T21:34:22.353 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@trial026.front.sepia.ceph.com' 2026-02-22T21:34:22.353 INFO:teuthology.run_tasks:Running task clock... 2026-02-22T21:34:22.365 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2026-02-22T21:34:22.365 INFO:teuthology.orchestra.run:Running command with timeout 360 2026-02-22T21:34:22.365 DEBUG:teuthology.orchestra.run.trial004:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2026-02-22T21:34:22.367 INFO:teuthology.orchestra.run:Running command with timeout 360 2026-02-22T21:34:22.367 DEBUG:teuthology.orchestra.run.trial026:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2026-02-22T21:34:22.393 INFO:teuthology.orchestra.run.trial004.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2026-02-22T21:34:22.404 INFO:teuthology.orchestra.run.trial004.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2026-02-22T21:34:22.421 INFO:teuthology.orchestra.run.trial026.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2026-02-22T21:34:22.424 INFO:teuthology.orchestra.run.trial004.stderr:sudo: ntpd: command not found 2026-02-22T21:34:22.433 INFO:teuthology.orchestra.run.trial026.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2026-02-22T21:34:22.434 INFO:teuthology.orchestra.run.trial004.stdout:506 Cannot talk to daemon 2026-02-22T21:34:22.446 INFO:teuthology.orchestra.run.trial004.stderr:Failed to start ntp.service: Unit ntp.service not found. 2026-02-22T21:34:22.455 INFO:teuthology.orchestra.run.trial026.stderr:sudo: ntpd: command not found 2026-02-22T21:34:22.458 INFO:teuthology.orchestra.run.trial004.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2026-02-22T21:34:22.465 INFO:teuthology.orchestra.run.trial026.stdout:506 Cannot talk to daemon 2026-02-22T21:34:22.477 INFO:teuthology.orchestra.run.trial026.stderr:Failed to start ntp.service: Unit ntp.service not found. 2026-02-22T21:34:22.488 INFO:teuthology.orchestra.run.trial026.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2026-02-22T21:34:22.508 INFO:teuthology.orchestra.run.trial004.stderr:bash: line 1: ntpq: command not found 2026-02-22T21:34:22.510 INFO:teuthology.orchestra.run.trial004.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2026-02-22T21:34:22.510 INFO:teuthology.orchestra.run.trial004.stdout:=============================================================================== 2026-02-22T21:34:22.541 INFO:teuthology.orchestra.run.trial026.stderr:bash: line 1: ntpq: command not found 2026-02-22T21:34:22.567 INFO:teuthology.orchestra.run.trial026.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2026-02-22T21:34:22.567 INFO:teuthology.orchestra.run.trial026.stdout:=============================================================================== 2026-02-22T21:34:22.567 INFO:teuthology.orchestra.run.trial026.stdout:^? 104-167-215-195.ipv4.ber> 0 6 0 - +0ns[ +0ns] +/- 0ns 2026-02-22T21:34:22.567 INFO:teuthology.orchestra.run.trial026.stdout:^? 23.186.168.129 0 6 0 - +0ns[ +0ns] +/- 0ns 2026-02-22T21:34:22.567 INFO:teuthology.orchestra.run.trial026.stdout:^? dns-e.ns4v.icu 0 6 0 - +0ns[ +0ns] +/- 0ns 2026-02-22T21:34:22.568 INFO:teuthology.orchestra.run.trial026.stdout:^? nyc3.us.ntp.li 0 6 0 - +0ns[ +0ns] +/- 0ns 2026-02-22T21:34:22.569 INFO:teuthology.run_tasks:Running task install... 2026-02-22T21:34:22.582 DEBUG:teuthology.task.install:project ceph 2026-02-22T21:34:22.582 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579'}} 2026-02-22T21:34:22.582 DEBUG:teuthology.task.install:config {'extra_packages': ['rbd-mirror'], 'flavor': 'default', 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579'} 2026-02-22T21:34:22.582 INFO:teuthology.task.install:Using flavor: default 2026-02-22T21:34:22.589 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'ceph-volume', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'ceph-volume', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2026-02-22T21:34:22.589 INFO:teuthology.task.install:extra packages: ['rbd-mirror'] 2026-02-22T21:34:22.590 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': None, 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': [], 'extra_packages': ['rbd-mirror'], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579', 'tag': None, 'wait_for_package': False} 2026-02-22T21:34:22.590 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=f90f4eaebb28b619cf76e80cbbe997656da69579 2026-02-22T21:34:22.592 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': None, 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': [], 'extra_packages': ['rbd-mirror'], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579', 'tag': None, 'wait_for_package': False} 2026-02-22T21:34:22.592 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=f90f4eaebb28b619cf76e80cbbe997656da69579 2026-02-22T21:34:22.783 INFO:teuthology.task.install.rpm:Pulling from https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/centos/9/flavors/default/ 2026-02-22T21:34:22.784 INFO:teuthology.task.install.rpm:Package version is 19.2.3-689.gf90f4eae 2026-02-22T21:34:22.793 INFO:teuthology.task.install.rpm:Pulling from https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/centos/9/flavors/default/ 2026-02-22T21:34:22.794 INFO:teuthology.task.install.rpm:Package version is 19.2.3-689.gf90f4eae 2026-02-22T21:34:22.938 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2026-02-22T21:34:22.938 DEBUG:teuthology.orchestra.run.trial004:> set -ex 2026-02-22T21:34:22.938 DEBUG:teuthology.orchestra.run.trial004:> sudo dd of=/etc/yum.repos.d/ceph.repo 2026-02-22T21:34:22.951 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2026-02-22T21:34:22.951 DEBUG:teuthology.orchestra.run.trial026:> set -ex 2026-02-22T21:34:22.951 DEBUG:teuthology.orchestra.run.trial026:> sudo dd of=/etc/yum.repos.d/ceph.repo 2026-02-22T21:34:22.963 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, ceph-volume, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd, rbd-mirror on remote rpm x86_64 2026-02-22T21:34:22.963 DEBUG:teuthology.orchestra.run.trial004:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;sha1/f90f4eaebb28b619cf76e80cbbe997656da69579/;g' /etc/yum.repos.d/ceph.repo ; fi 2026-02-22T21:34:22.977 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, ceph-volume, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd, rbd-mirror on remote rpm x86_64 2026-02-22T21:34:22.978 DEBUG:teuthology.orchestra.run.trial026:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;sha1/f90f4eaebb28b619cf76e80cbbe997656da69579/;g' /etc/yum.repos.d/ceph.repo ; fi 2026-02-22T21:34:23.027 DEBUG:teuthology.orchestra.run.trial004:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2026-02-22T21:34:23.042 DEBUG:teuthology.orchestra.run.trial026:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2026-02-22T21:34:23.100 DEBUG:teuthology.orchestra.run.trial004:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2026-02-22T21:34:23.116 DEBUG:teuthology.orchestra.run.trial026:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2026-02-22T21:34:23.164 INFO:teuthology.orchestra.run.trial004.stdout:check_obsoletes = 1 2026-02-22T21:34:23.165 DEBUG:teuthology.orchestra.run.trial004:> sudo yum clean all 2026-02-22T21:34:23.179 INFO:teuthology.orchestra.run.trial026.stdout:check_obsoletes = 1 2026-02-22T21:34:23.180 DEBUG:teuthology.orchestra.run.trial026:> sudo yum clean all 2026-02-22T21:34:23.325 INFO:teuthology.orchestra.run.trial004.stdout:41 files removed 2026-02-22T21:34:23.340 DEBUG:teuthology.orchestra.run.trial004:> sudo yum -y install ceph-radosgw 2026-02-22T21:34:23.342 INFO:teuthology.orchestra.run.trial026.stdout:41 files removed 2026-02-22T21:34:23.356 DEBUG:teuthology.orchestra.run.trial026:> sudo yum -y install ceph-radosgw 2026-02-22T21:34:23.611 INFO:teuthology.orchestra.run.trial004.stdout:ceph packages for x86_64 665 kB/s | 86 kB 00:00 2026-02-22T21:34:23.631 INFO:teuthology.orchestra.run.trial026.stdout:ceph packages for x86_64 648 kB/s | 86 kB 00:00 2026-02-22T21:34:23.748 INFO:teuthology.orchestra.run.trial004.stdout:ceph noarch packages 107 kB/s | 12 kB 00:00 2026-02-22T21:34:23.762 INFO:teuthology.orchestra.run.trial026.stdout:ceph noarch packages 112 kB/s | 12 kB 00:00 2026-02-22T21:34:23.876 INFO:teuthology.orchestra.run.trial004.stdout:ceph source packages 18 kB/s | 1.9 kB 00:00 2026-02-22T21:34:23.887 INFO:teuthology.orchestra.run.trial026.stdout:ceph source packages 18 kB/s | 1.9 kB 00:00 2026-02-22T21:34:24.140 INFO:teuthology.orchestra.run.trial004.stdout:CentOS Stream 9 - BaseOS 37 MB/s | 8.9 MB 00:00 2026-02-22T21:34:25.062 INFO:teuthology.orchestra.run.trial026.stdout:CentOS Stream 9 - BaseOS 7.7 MB/s | 8.9 MB 00:01 2026-02-22T21:34:27.381 INFO:teuthology.orchestra.run.trial004.stdout:CentOS Stream 9 - AppStream 9.9 MB/s | 27 MB 00:02 2026-02-22T21:34:30.727 INFO:teuthology.orchestra.run.trial004.stdout:CentOS Stream 9 - CRB 7.6 MB/s | 7.8 MB 00:01 2026-02-22T21:34:31.611 INFO:teuthology.orchestra.run.trial004.stdout:CentOS Stream 9 - Extras packages 119 kB/s | 20 kB 00:00 2026-02-22T21:34:31.955 INFO:teuthology.orchestra.run.trial004.stdout:Extra Packages for Enterprise Linux 72 MB/s | 20 MB 00:00 2026-02-22T21:34:35.127 INFO:teuthology.orchestra.run.trial004.stdout:lab-extras 2.2 MB/s | 50 kB 00:00 2026-02-22T21:34:36.335 INFO:teuthology.orchestra.run.trial004.stdout:Dependencies resolved. 2026-02-22T21:34:36.336 INFO:teuthology.orchestra.run.trial004.stdout:================================================================================ 2026-02-22T21:34:36.336 INFO:teuthology.orchestra.run.trial004.stdout: Package Arch Version Repository Size 2026-02-22T21:34:36.336 INFO:teuthology.orchestra.run.trial004.stdout:================================================================================ 2026-02-22T21:34:36.336 INFO:teuthology.orchestra.run.trial004.stdout:Installing: 2026-02-22T21:34:36.336 INFO:teuthology.orchestra.run.trial004.stdout: ceph-radosgw x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 11 M 2026-02-22T21:34:36.336 INFO:teuthology.orchestra.run.trial004.stdout:Upgrading: 2026-02-22T21:34:36.336 INFO:teuthology.orchestra.run.trial004.stdout: librados2 x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 3.4 M 2026-02-22T21:34:36.336 INFO:teuthology.orchestra.run.trial004.stdout: librbd1 x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 3.2 M 2026-02-22T21:34:36.336 INFO:teuthology.orchestra.run.trial004.stdout:Installing dependencies: 2026-02-22T21:34:36.336 INFO:teuthology.orchestra.run.trial004.stdout: boost-program-options x86_64 1.75.0-13.el9 appstream 104 k 2026-02-22T21:34:36.336 INFO:teuthology.orchestra.run.trial004.stdout: ceph-base x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 5.5 M 2026-02-22T21:34:36.336 INFO:teuthology.orchestra.run.trial004.stdout: ceph-common x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 22 M 2026-02-22T21:34:36.337 INFO:teuthology.orchestra.run.trial004.stdout: ceph-selinux x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 25 k 2026-02-22T21:34:36.337 INFO:teuthology.orchestra.run.trial004.stdout: gperftools-libs x86_64 2.9.1-3.el9 epel 308 k 2026-02-22T21:34:36.337 INFO:teuthology.orchestra.run.trial004.stdout: libarrow x86_64 9.0.0-15.el9 epel 4.4 M 2026-02-22T21:34:36.337 INFO:teuthology.orchestra.run.trial004.stdout: libarrow-doc noarch 9.0.0-15.el9 epel 25 k 2026-02-22T21:34:36.337 INFO:teuthology.orchestra.run.trial004.stdout: libcephfs2 x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 999 k 2026-02-22T21:34:36.337 INFO:teuthology.orchestra.run.trial004.stdout: libnbd x86_64 1.20.3-4.el9 appstream 164 k 2026-02-22T21:34:36.337 INFO:teuthology.orchestra.run.trial004.stdout: liboath x86_64 2.6.12-1.el9 epel 49 k 2026-02-22T21:34:36.337 INFO:teuthology.orchestra.run.trial004.stdout: libpmemobj x86_64 1.12.1-1.el9 appstream 160 k 2026-02-22T21:34:36.337 INFO:teuthology.orchestra.run.trial004.stdout: librabbitmq x86_64 0.11.0-7.el9 appstream 45 k 2026-02-22T21:34:36.337 INFO:teuthology.orchestra.run.trial004.stdout: libradosstriper1 x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 503 k 2026-02-22T21:34:36.337 INFO:teuthology.orchestra.run.trial004.stdout: librdkafka x86_64 1.6.1-102.el9 appstream 662 k 2026-02-22T21:34:36.337 INFO:teuthology.orchestra.run.trial004.stdout: librgw2 x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 5.4 M 2026-02-22T21:34:36.337 INFO:teuthology.orchestra.run.trial004.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2026-02-22T21:34:36.337 INFO:teuthology.orchestra.run.trial004.stdout: lttng-ust x86_64 2.12.0-6.el9 appstream 292 k 2026-02-22T21:34:36.337 INFO:teuthology.orchestra.run.trial004.stdout: mailcap noarch 2.1.49-5.el9 baseos 33 k 2026-02-22T21:34:36.337 INFO:teuthology.orchestra.run.trial004.stdout: parquet-libs x86_64 9.0.0-15.el9 epel 838 k 2026-02-22T21:34:36.337 INFO:teuthology.orchestra.run.trial004.stdout: pciutils x86_64 3.7.0-7.el9 baseos 93 k 2026-02-22T21:34:36.337 INFO:teuthology.orchestra.run.trial004.stdout: python3-ceph-argparse x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 45 k 2026-02-22T21:34:36.337 INFO:teuthology.orchestra.run.trial004.stdout: python3-ceph-common x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 142 k 2026-02-22T21:34:36.337 INFO:teuthology.orchestra.run.trial004.stdout: python3-cephfs x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 165 k 2026-02-22T21:34:36.337 INFO:teuthology.orchestra.run.trial004.stdout: python3-rados x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 322 k 2026-02-22T21:34:36.337 INFO:teuthology.orchestra.run.trial004.stdout: python3-rbd x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 303 k 2026-02-22T21:34:36.337 INFO:teuthology.orchestra.run.trial004.stdout: python3-rgw x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 100 k 2026-02-22T21:34:36.337 INFO:teuthology.orchestra.run.trial004.stdout: qatlib x86_64 25.08.0-2.el9 appstream 240 k 2026-02-22T21:34:36.337 INFO:teuthology.orchestra.run.trial004.stdout: qatzip-libs x86_64 1.3.1-1.el9 appstream 66 k 2026-02-22T21:34:36.337 INFO:teuthology.orchestra.run.trial004.stdout: re2 x86_64 1:20211101-20.el9 epel 191 k 2026-02-22T21:34:36.337 INFO:teuthology.orchestra.run.trial004.stdout: thrift x86_64 0.15.0-4.el9 epel 1.6 M 2026-02-22T21:34:36.337 INFO:teuthology.orchestra.run.trial004.stdout:Installing weak dependencies: 2026-02-22T21:34:36.337 INFO:teuthology.orchestra.run.trial004.stdout: nvme-cli x86_64 2.16-1.el9 baseos 1.2 M 2026-02-22T21:34:36.337 INFO:teuthology.orchestra.run.trial004.stdout: qatlib-service x86_64 25.08.0-2.el9 appstream 37 k 2026-02-22T21:34:36.338 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:34:36.338 INFO:teuthology.orchestra.run.trial004.stdout:Transaction Summary 2026-02-22T21:34:36.338 INFO:teuthology.orchestra.run.trial004.stdout:================================================================================ 2026-02-22T21:34:36.338 INFO:teuthology.orchestra.run.trial004.stdout:Install 33 Packages 2026-02-22T21:34:36.338 INFO:teuthology.orchestra.run.trial004.stdout:Upgrade 2 Packages 2026-02-22T21:34:36.338 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:34:36.338 INFO:teuthology.orchestra.run.trial004.stdout:Total download size: 63 M 2026-02-22T21:34:36.338 INFO:teuthology.orchestra.run.trial004.stdout:Downloading Packages: 2026-02-22T21:34:37.016 INFO:teuthology.orchestra.run.trial004.stdout:(1/35): ceph-base-19.2.3-689.gf90f4eae.el9.x86_ 13 MB/s | 5.5 MB 00:00 2026-02-22T21:34:37.058 INFO:teuthology.orchestra.run.trial004.stdout:(2/35): ceph-selinux-19.2.3-689.gf90f4eae.el9.x 601 kB/s | 25 kB 00:00 2026-02-22T21:34:37.101 INFO:teuthology.orchestra.run.trial004.stdout:(3/35): ceph-radosgw-19.2.3-689.gf90f4eae.el9.x 21 MB/s | 11 MB 00:00 2026-02-22T21:34:37.106 INFO:teuthology.orchestra.run.trial004.stdout:(4/35): libcephfs2-19.2.3-689.gf90f4eae.el9.x86 20 MB/s | 999 kB 00:00 2026-02-22T21:34:37.135 INFO:teuthology.orchestra.run.trial004.stdout:(5/35): libradosstriper1-19.2.3-689.gf90f4eae.e 14 MB/s | 503 kB 00:00 2026-02-22T21:34:37.177 INFO:teuthology.orchestra.run.trial004.stdout:(6/35): python3-ceph-argparse-19.2.3-689.gf90f4 1.1 MB/s | 45 kB 00:00 2026-02-22T21:34:37.221 INFO:teuthology.orchestra.run.trial004.stdout:(7/35): python3-ceph-common-19.2.3-689.gf90f4ea 3.2 MB/s | 142 kB 00:00 2026-02-22T21:34:37.269 INFO:teuthology.orchestra.run.trial004.stdout:(8/35): python3-cephfs-19.2.3-689.gf90f4eae.el9 3.4 MB/s | 165 kB 00:00 2026-02-22T21:34:37.322 INFO:teuthology.orchestra.run.trial004.stdout:(9/35): python3-rados-19.2.3-689.gf90f4eae.el9. 5.9 MB/s | 322 kB 00:00 2026-02-22T21:34:37.343 INFO:teuthology.orchestra.run.trial004.stdout:(10/35): librgw2-19.2.3-689.gf90f4eae.el9.x86_6 23 MB/s | 5.4 MB 00:00 2026-02-22T21:34:37.368 INFO:teuthology.orchestra.run.trial004.stdout:(11/35): python3-rbd-19.2.3-689.gf90f4eae.el9.x 6.5 MB/s | 303 kB 00:00 2026-02-22T21:34:37.371 INFO:teuthology.orchestra.run.trial004.stdout:(12/35): python3-rgw-19.2.3-689.gf90f4eae.el9.x 3.4 MB/s | 100 kB 00:00 2026-02-22T21:34:37.456 INFO:teuthology.orchestra.run.trial004.stdout:(13/35): mailcap-2.1.49-5.el9.noarch.rpm 379 kB/s | 33 kB 00:00 2026-02-22T21:34:37.488 INFO:teuthology.orchestra.run.trial004.stdout:(14/35): ceph-common-19.2.3-689.gf90f4eae.el9.x 24 MB/s | 22 MB 00:00 2026-02-22T21:34:37.531 INFO:teuthology.orchestra.run.trial004.stdout:(15/35): pciutils-3.7.0-7.el9.x86_64.rpm 1.2 MB/s | 93 kB 00:00 2026-02-22T21:34:37.616 INFO:teuthology.orchestra.run.trial004.stdout:(16/35): nvme-cli-2.16-1.el9.x86_64.rpm 4.7 MB/s | 1.2 MB 00:00 2026-02-22T21:34:38.176 INFO:teuthology.orchestra.run.trial004.stdout:(17/35): boost-program-options-1.75.0-13.el9.x8 151 kB/s | 104 kB 00:00 2026-02-22T21:34:38.259 INFO:teuthology.orchestra.run.trial004.stdout:(18/35): libnbd-1.20.3-4.el9.x86_64.rpm 225 kB/s | 164 kB 00:00 2026-02-22T21:34:38.347 INFO:teuthology.orchestra.run.trial004.stdout:(19/35): libpmemobj-1.12.1-1.el9.x86_64.rpm 219 kB/s | 160 kB 00:00 2026-02-22T21:34:38.777 INFO:teuthology.orchestra.run.trial004.stdout:(20/35): librabbitmq-0.11.0-7.el9.x86_64.rpm 75 kB/s | 45 kB 00:00 2026-02-22T21:34:39.138 INFO:teuthology.orchestra.run.trial004.stdout:(21/35): librdkafka-1.6.1-102.el9.x86_64.rpm 754 kB/s | 662 kB 00:00 2026-02-22T21:34:39.139 INFO:teuthology.orchestra.run.trial004.stdout:(22/35): lttng-ust-2.12.0-6.el9.x86_64.rpm 369 kB/s | 292 kB 00:00 2026-02-22T21:34:39.543 INFO:teuthology.orchestra.run.trial004.stdout:(23/35): qatlib-25.08.0-2.el9.x86_64.rpm 313 kB/s | 240 kB 00:00 2026-02-22T21:34:39.713 INFO:teuthology.orchestra.run.trial004.stdout:(24/35): qatlib-service-25.08.0-2.el9.x86_64.rp 64 kB/s | 37 kB 00:00 2026-02-22T21:34:39.773 INFO:teuthology.orchestra.run.trial004.stdout:(25/35): qatzip-libs-1.3.1-1.el9.x86_64.rpm 105 kB/s | 66 kB 00:00 2026-02-22T21:34:39.924 INFO:teuthology.orchestra.run.trial004.stdout:(26/35): gperftools-libs-2.9.1-3.el9.x86_64.rpm 806 kB/s | 308 kB 00:00 2026-02-22T21:34:39.982 INFO:teuthology.orchestra.run.trial004.stdout:(27/35): liboath-2.6.12-1.el9.x86_64.rpm 848 kB/s | 49 kB 00:00 2026-02-22T21:34:39.999 INFO:teuthology.orchestra.run.trial004.stdout:(28/35): libarrow-doc-9.0.0-15.el9.noarch.rpm 110 kB/s | 25 kB 00:00 2026-02-22T21:34:40.040 INFO:teuthology.orchestra.run.trial004.stdout:(29/35): libunwind-1.6.2-1.el9.x86_64.rpm 1.1 MB/s | 67 kB 00:00 2026-02-22T21:34:40.117 INFO:teuthology.orchestra.run.trial004.stdout:(30/35): re2-20211101-20.el9.x86_64.rpm 2.4 MB/s | 191 kB 00:00 2026-02-22T21:34:40.284 INFO:teuthology.orchestra.run.trial004.stdout:(31/35): parquet-libs-9.0.0-15.el9.x86_64.rpm 2.9 MB/s | 838 kB 00:00 2026-02-22T21:34:40.318 INFO:teuthology.orchestra.run.trial004.stdout:(32/35): libarrow-9.0.0-15.el9.x86_64.rpm 7.3 MB/s | 4.4 MB 00:00 2026-02-22T21:34:40.322 INFO:teuthology.orchestra.run.trial004.stdout:(33/35): thrift-0.15.0-4.el9.x86_64.rpm 7.7 MB/s | 1.6 MB 00:00 2026-02-22T21:34:40.406 INFO:teuthology.orchestra.run.trial004.stdout:(34/35): librados2-19.2.3-689.gf90f4eae.el9.x86 28 MB/s | 3.4 MB 00:00 2026-02-22T21:34:40.454 INFO:teuthology.orchestra.run.trial004.stdout:(35/35): librbd1-19.2.3-689.gf90f4eae.el9.x86_6 23 MB/s | 3.2 MB 00:00 2026-02-22T21:34:40.456 INFO:teuthology.orchestra.run.trial004.stdout:-------------------------------------------------------------------------------- 2026-02-22T21:34:40.457 INFO:teuthology.orchestra.run.trial004.stdout:Total 15 MB/s | 63 MB 00:04 2026-02-22T21:34:40.554 INFO:teuthology.orchestra.run.trial004.stdout:Running transaction check 2026-02-22T21:34:40.573 INFO:teuthology.orchestra.run.trial004.stdout:Transaction check succeeded. 2026-02-22T21:34:40.573 INFO:teuthology.orchestra.run.trial004.stdout:Running transaction test 2026-02-22T21:34:40.827 INFO:teuthology.orchestra.run.trial004.stdout:Transaction test succeeded. 2026-02-22T21:34:40.828 INFO:teuthology.orchestra.run.trial004.stdout:Running transaction 2026-02-22T21:34:41.120 INFO:teuthology.orchestra.run.trial004.stdout: Preparing : 1/1 2026-02-22T21:34:41.129 INFO:teuthology.orchestra.run.trial004.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 1/37 2026-02-22T21:34:41.132 INFO:teuthology.orchestra.run.trial004.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 2/37 2026-02-22T21:34:41.143 INFO:teuthology.orchestra.run.trial004.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 3/37 2026-02-22T21:34:41.295 INFO:teuthology.orchestra.run.trial004.stdout: Installing : thrift-0.15.0-4.el9.x86_64 4/37 2026-02-22T21:34:41.299 INFO:teuthology.orchestra.run.trial004.stdout: Upgrading : librados2-2:19.2.3-689.gf90f4eae.el9.x86_64 5/37 2026-02-22T21:34:41.327 INFO:teuthology.orchestra.run.trial004.stdout: Running scriptlet: librados2-2:19.2.3-689.gf90f4eae.el9.x86_64 5/37 2026-02-22T21:34:41.370 INFO:teuthology.orchestra.run.trial004.stdout: Installing : python3-rados-2:19.2.3-689.gf90f4eae.el9.x86_64 6/37 2026-02-22T21:34:41.371 INFO:teuthology.orchestra.run.trial004.stdout: Installing : libcephfs2-2:19.2.3-689.gf90f4eae.el9.x86_64 7/37 2026-02-22T21:34:41.381 INFO:teuthology.orchestra.run.trial004.stdout: Running scriptlet: libcephfs2-2:19.2.3-689.gf90f4eae.el9.x86_64 7/37 2026-02-22T21:34:41.385 INFO:teuthology.orchestra.run.trial004.stdout: Installing : liboath-2.6.12-1.el9.x86_64 8/37 2026-02-22T21:34:41.394 INFO:teuthology.orchestra.run.trial004.stdout: Installing : python3-ceph-argparse-2:19.2.3-689.gf90f4eae.el9.x 9/37 2026-02-22T21:34:41.417 INFO:teuthology.orchestra.run.trial004.stdout: Installing : python3-cephfs-2:19.2.3-689.gf90f4eae.el9.x86_64 10/37 2026-02-22T21:34:41.417 INFO:teuthology.orchestra.run.trial004.stdout: Installing : libradosstriper1-2:19.2.3-689.gf90f4eae.el9.x86_64 11/37 2026-02-22T21:34:41.429 INFO:teuthology.orchestra.run.trial004.stdout: Running scriptlet: libradosstriper1-2:19.2.3-689.gf90f4eae.el9.x86_64 11/37 2026-02-22T21:34:41.461 INFO:teuthology.orchestra.run.trial004.stdout: Installing : re2-1:20211101-20.el9.x86_64 12/37 2026-02-22T21:34:41.466 INFO:teuthology.orchestra.run.trial004.stdout: Installing : libarrow-9.0.0-15.el9.x86_64 13/37 2026-02-22T21:34:41.473 INFO:teuthology.orchestra.run.trial004.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 14/37 2026-02-22T21:34:41.475 INFO:teuthology.orchestra.run.trial004.stdout: Installing : gperftools-libs-2.9.1-3.el9.x86_64 15/37 2026-02-22T21:34:41.477 INFO:teuthology.orchestra.run.trial004.stdout: Installing : libarrow-doc-9.0.0-15.el9.noarch 16/37 2026-02-22T21:34:41.483 INFO:teuthology.orchestra.run.trial004.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 17/37 2026-02-22T21:34:41.622 INFO:teuthology.orchestra.run.trial004.stdout: Installing : libnbd-1.20.3-4.el9.x86_64 18/37 2026-02-22T21:34:41.623 INFO:teuthology.orchestra.run.trial004.stdout: Upgrading : librbd1-2:19.2.3-689.gf90f4eae.el9.x86_64 19/37 2026-02-22T21:34:41.649 INFO:teuthology.orchestra.run.trial004.stdout: Running scriptlet: librbd1-2:19.2.3-689.gf90f4eae.el9.x86_64 19/37 2026-02-22T21:34:41.651 INFO:teuthology.orchestra.run.trial004.stdout: Installing : python3-rbd-2:19.2.3-689.gf90f4eae.el9.x86_64 20/37 2026-02-22T21:34:41.657 INFO:teuthology.orchestra.run.trial004.stdout: Installing : boost-program-options-1.75.0-13.el9.x86_64 21/37 2026-02-22T21:34:41.882 INFO:teuthology.orchestra.run.trial004.stdout: Installing : parquet-libs-9.0.0-15.el9.x86_64 22/37 2026-02-22T21:34:41.883 INFO:teuthology.orchestra.run.trial004.stdout: Installing : librgw2-2:19.2.3-689.gf90f4eae.el9.x86_64 23/37 2026-02-22T21:34:41.899 INFO:teuthology.orchestra.run.trial004.stdout: Running scriptlet: librgw2-2:19.2.3-689.gf90f4eae.el9.x86_64 23/37 2026-02-22T21:34:41.903 INFO:teuthology.orchestra.run.trial004.stdout: Installing : python3-rgw-2:19.2.3-689.gf90f4eae.el9.x86_64 24/37 2026-02-22T21:34:41.909 INFO:teuthology.orchestra.run.trial004.stdout: Installing : pciutils-3.7.0-7.el9.x86_64 25/37 2026-02-22T21:34:41.912 INFO:teuthology.orchestra.run.trial004.stdout: Installing : qatlib-25.08.0-2.el9.x86_64 26/37 2026-02-22T21:34:41.912 INFO:teuthology.orchestra.run.trial004.stdout: Installing : qatlib-service-25.08.0-2.el9.x86_64 27/37 2026-02-22T21:34:41.930 INFO:teuthology.orchestra.run.trial004.stdout: Running scriptlet: qatlib-service-25.08.0-2.el9.x86_64 27/37 2026-02-22T21:34:42.046 INFO:teuthology.orchestra.run.trial004.stdout: Installing : qatzip-libs-1.3.1-1.el9.x86_64 28/37 2026-02-22T21:34:42.048 INFO:teuthology.orchestra.run.trial004.stdout: Installing : nvme-cli-2.16-1.el9.x86_64 29/37 2026-02-22T21:34:42.331 INFO:teuthology.orchestra.run.trial004.stdout: Running scriptlet: nvme-cli-2.16-1.el9.x86_64 29/37 2026-02-22T21:34:42.331 INFO:teuthology.orchestra.run.trial004.stdout:Created symlink /etc/systemd/system/default.target.wants/nvmefc-boot-connections.service → /usr/lib/systemd/system/nvmefc-boot-connections.service. 2026-02-22T21:34:42.332 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:34:42.369 INFO:teuthology.orchestra.run.trial004.stdout: Installing : mailcap-2.1.49-5.el9.noarch 30/37 2026-02-22T21:34:42.371 INFO:teuthology.orchestra.run.trial004.stdout: Installing : python3-ceph-common-2:19.2.3-689.gf90f4eae.el9.x86 31/37 2026-02-22T21:34:43.350 INFO:teuthology.orchestra.run.trial004.stdout: Running scriptlet: ceph-common-2:19.2.3-689.gf90f4eae.el9.x86_64 32/37 2026-02-22T21:34:43.353 INFO:teuthology.orchestra.run.trial004.stdout: Installing : ceph-common-2:19.2.3-689.gf90f4eae.el9.x86_64 32/37 2026-02-22T21:34:43.635 INFO:teuthology.orchestra.run.trial004.stdout: Running scriptlet: ceph-common-2:19.2.3-689.gf90f4eae.el9.x86_64 32/37 2026-02-22T21:34:43.639 INFO:teuthology.orchestra.run.trial004.stdout: Installing : ceph-base-2:19.2.3-689.gf90f4eae.el9.x86_64 33/37 2026-02-22T21:34:43.678 INFO:teuthology.orchestra.run.trial004.stdout: Running scriptlet: ceph-base-2:19.2.3-689.gf90f4eae.el9.x86_64 33/37 2026-02-22T21:34:43.678 INFO:teuthology.orchestra.run.trial004.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2026-02-22T21:34:43.678 INFO:teuthology.orchestra.run.trial004.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2026-02-22T21:34:43.678 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:34:43.681 INFO:teuthology.orchestra.run.trial004.stdout: Installing : ceph-selinux-2:19.2.3-689.gf90f4eae.el9.x86_64 34/37 2026-02-22T21:34:48.181 INFO:teuthology.orchestra.run.trial004.stdout: Running scriptlet: ceph-selinux-2:19.2.3-689.gf90f4eae.el9.x86_64 34/37 2026-02-22T21:34:48.182 INFO:teuthology.orchestra.run.trial004.stdout:skipping the directory /sys 2026-02-22T21:34:48.182 INFO:teuthology.orchestra.run.trial004.stdout:skipping the directory /proc 2026-02-22T21:34:48.182 INFO:teuthology.orchestra.run.trial004.stdout:skipping the directory /mnt 2026-02-22T21:34:48.182 INFO:teuthology.orchestra.run.trial004.stdout:skipping the directory /var/tmp 2026-02-22T21:34:48.182 INFO:teuthology.orchestra.run.trial004.stdout:skipping the directory /home 2026-02-22T21:34:48.182 INFO:teuthology.orchestra.run.trial004.stdout:skipping the directory /root 2026-02-22T21:34:48.182 INFO:teuthology.orchestra.run.trial004.stdout:skipping the directory /tmp 2026-02-22T21:34:48.182 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:34:48.633 INFO:teuthology.orchestra.run.trial004.stdout: Installing : ceph-radosgw-2:19.2.3-689.gf90f4eae.el9.x86_64 35/37 2026-02-22T21:34:48.655 INFO:teuthology.orchestra.run.trial004.stdout: Running scriptlet: ceph-radosgw-2:19.2.3-689.gf90f4eae.el9.x86_64 35/37 2026-02-22T21:34:48.656 INFO:teuthology.orchestra.run.trial004.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-22T21:34:48.656 INFO:teuthology.orchestra.run.trial004.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2026-02-22T21:34:48.656 INFO:teuthology.orchestra.run.trial004.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2026-02-22T21:34:48.656 INFO:teuthology.orchestra.run.trial004.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2026-02-22T21:34:48.656 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:34:48.657 INFO:teuthology.orchestra.run.trial004.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 36/37 2026-02-22T21:34:48.669 INFO:teuthology.orchestra.run.trial004.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 36/37 2026-02-22T21:34:48.669 INFO:teuthology.orchestra.run.trial004.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 37/37 2026-02-22T21:34:48.723 INFO:teuthology.orchestra.run.trial004.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 37/37 2026-02-22T21:34:48.724 INFO:teuthology.orchestra.run.trial004.stdout:Creating group 'qat' with GID 992. 2026-02-22T21:34:48.724 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:34:48.997 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : ceph-base-2:19.2.3-689.gf90f4eae.el9.x86_64 1/37 2026-02-22T21:34:48.997 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : ceph-common-2:19.2.3-689.gf90f4eae.el9.x86_64 2/37 2026-02-22T21:34:48.997 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : ceph-radosgw-2:19.2.3-689.gf90f4eae.el9.x86_64 3/37 2026-02-22T21:34:48.997 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : ceph-selinux-2:19.2.3-689.gf90f4eae.el9.x86_64 4/37 2026-02-22T21:34:48.997 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : libcephfs2-2:19.2.3-689.gf90f4eae.el9.x86_64 5/37 2026-02-22T21:34:48.997 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : libradosstriper1-2:19.2.3-689.gf90f4eae.el9.x86_64 6/37 2026-02-22T21:34:48.997 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : librgw2-2:19.2.3-689.gf90f4eae.el9.x86_64 7/37 2026-02-22T21:34:48.997 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : python3-ceph-argparse-2:19.2.3-689.gf90f4eae.el9.x 8/37 2026-02-22T21:34:48.997 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : python3-ceph-common-2:19.2.3-689.gf90f4eae.el9.x86 9/37 2026-02-22T21:34:48.997 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : python3-cephfs-2:19.2.3-689.gf90f4eae.el9.x86_64 10/37 2026-02-22T21:34:48.997 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : python3-rados-2:19.2.3-689.gf90f4eae.el9.x86_64 11/37 2026-02-22T21:34:48.997 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : python3-rbd-2:19.2.3-689.gf90f4eae.el9.x86_64 12/37 2026-02-22T21:34:48.997 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : python3-rgw-2:19.2.3-689.gf90f4eae.el9.x86_64 13/37 2026-02-22T21:34:48.997 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : mailcap-2.1.49-5.el9.noarch 14/37 2026-02-22T21:34:48.998 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : nvme-cli-2.16-1.el9.x86_64 15/37 2026-02-22T21:34:48.998 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : pciutils-3.7.0-7.el9.x86_64 16/37 2026-02-22T21:34:48.998 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : boost-program-options-1.75.0-13.el9.x86_64 17/37 2026-02-22T21:34:48.998 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : libnbd-1.20.3-4.el9.x86_64 18/37 2026-02-22T21:34:48.998 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 19/37 2026-02-22T21:34:48.998 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 20/37 2026-02-22T21:34:48.998 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 21/37 2026-02-22T21:34:48.998 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 22/37 2026-02-22T21:34:48.998 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : qatlib-25.08.0-2.el9.x86_64 23/37 2026-02-22T21:34:48.998 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : qatlib-service-25.08.0-2.el9.x86_64 24/37 2026-02-22T21:34:48.998 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : qatzip-libs-1.3.1-1.el9.x86_64 25/37 2026-02-22T21:34:48.998 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : gperftools-libs-2.9.1-3.el9.x86_64 26/37 2026-02-22T21:34:48.998 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : libarrow-9.0.0-15.el9.x86_64 27/37 2026-02-22T21:34:48.998 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : libarrow-doc-9.0.0-15.el9.noarch 28/37 2026-02-22T21:34:48.998 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : liboath-2.6.12-1.el9.x86_64 29/37 2026-02-22T21:34:48.998 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 30/37 2026-02-22T21:34:48.998 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : parquet-libs-9.0.0-15.el9.x86_64 31/37 2026-02-22T21:34:48.998 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : re2-1:20211101-20.el9.x86_64 32/37 2026-02-22T21:34:48.998 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : thrift-0.15.0-4.el9.x86_64 33/37 2026-02-22T21:34:48.998 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : librados2-2:19.2.3-689.gf90f4eae.el9.x86_64 34/37 2026-02-22T21:34:48.998 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 35/37 2026-02-22T21:34:48.998 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : librbd1-2:19.2.3-689.gf90f4eae.el9.x86_64 36/37 2026-02-22T21:34:49.059 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 37/37 2026-02-22T21:34:49.059 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:34:49.059 INFO:teuthology.orchestra.run.trial004.stdout:Upgraded: 2026-02-22T21:34:49.059 INFO:teuthology.orchestra.run.trial004.stdout: librados2-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:34:49.059 INFO:teuthology.orchestra.run.trial004.stdout: librbd1-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:34:49.059 INFO:teuthology.orchestra.run.trial004.stdout:Installed: 2026-02-22T21:34:49.059 INFO:teuthology.orchestra.run.trial004.stdout: boost-program-options-1.75.0-13.el9.x86_64 2026-02-22T21:34:49.059 INFO:teuthology.orchestra.run.trial004.stdout: ceph-base-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:34:49.059 INFO:teuthology.orchestra.run.trial004.stdout: ceph-common-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:34:49.060 INFO:teuthology.orchestra.run.trial004.stdout: ceph-radosgw-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:34:49.060 INFO:teuthology.orchestra.run.trial004.stdout: ceph-selinux-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:34:49.060 INFO:teuthology.orchestra.run.trial004.stdout: gperftools-libs-2.9.1-3.el9.x86_64 2026-02-22T21:34:49.060 INFO:teuthology.orchestra.run.trial004.stdout: libarrow-9.0.0-15.el9.x86_64 2026-02-22T21:34:49.060 INFO:teuthology.orchestra.run.trial004.stdout: libarrow-doc-9.0.0-15.el9.noarch 2026-02-22T21:34:49.060 INFO:teuthology.orchestra.run.trial004.stdout: libcephfs2-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:34:49.060 INFO:teuthology.orchestra.run.trial004.stdout: libnbd-1.20.3-4.el9.x86_64 2026-02-22T21:34:49.060 INFO:teuthology.orchestra.run.trial004.stdout: liboath-2.6.12-1.el9.x86_64 2026-02-22T21:34:49.060 INFO:teuthology.orchestra.run.trial004.stdout: libpmemobj-1.12.1-1.el9.x86_64 2026-02-22T21:34:49.060 INFO:teuthology.orchestra.run.trial004.stdout: librabbitmq-0.11.0-7.el9.x86_64 2026-02-22T21:34:49.060 INFO:teuthology.orchestra.run.trial004.stdout: libradosstriper1-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:34:49.060 INFO:teuthology.orchestra.run.trial004.stdout: librdkafka-1.6.1-102.el9.x86_64 2026-02-22T21:34:49.060 INFO:teuthology.orchestra.run.trial004.stdout: librgw2-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:34:49.060 INFO:teuthology.orchestra.run.trial004.stdout: libunwind-1.6.2-1.el9.x86_64 2026-02-22T21:34:49.060 INFO:teuthology.orchestra.run.trial004.stdout: lttng-ust-2.12.0-6.el9.x86_64 2026-02-22T21:34:49.060 INFO:teuthology.orchestra.run.trial004.stdout: mailcap-2.1.49-5.el9.noarch 2026-02-22T21:34:49.060 INFO:teuthology.orchestra.run.trial004.stdout: nvme-cli-2.16-1.el9.x86_64 2026-02-22T21:34:49.060 INFO:teuthology.orchestra.run.trial004.stdout: parquet-libs-9.0.0-15.el9.x86_64 2026-02-22T21:34:49.060 INFO:teuthology.orchestra.run.trial004.stdout: pciutils-3.7.0-7.el9.x86_64 2026-02-22T21:34:49.060 INFO:teuthology.orchestra.run.trial004.stdout: python3-ceph-argparse-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:34:49.060 INFO:teuthology.orchestra.run.trial004.stdout: python3-ceph-common-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:34:49.060 INFO:teuthology.orchestra.run.trial004.stdout: python3-cephfs-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:34:49.060 INFO:teuthology.orchestra.run.trial004.stdout: python3-rados-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:34:49.060 INFO:teuthology.orchestra.run.trial004.stdout: python3-rbd-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:34:49.060 INFO:teuthology.orchestra.run.trial004.stdout: python3-rgw-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:34:49.060 INFO:teuthology.orchestra.run.trial004.stdout: qatlib-25.08.0-2.el9.x86_64 2026-02-22T21:34:49.060 INFO:teuthology.orchestra.run.trial004.stdout: qatlib-service-25.08.0-2.el9.x86_64 2026-02-22T21:34:49.060 INFO:teuthology.orchestra.run.trial004.stdout: qatzip-libs-1.3.1-1.el9.x86_64 2026-02-22T21:34:49.060 INFO:teuthology.orchestra.run.trial004.stdout: re2-1:20211101-20.el9.x86_64 2026-02-22T21:34:49.061 INFO:teuthology.orchestra.run.trial004.stdout: thrift-0.15.0-4.el9.x86_64 2026-02-22T21:34:49.061 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:34:49.061 INFO:teuthology.orchestra.run.trial004.stdout:Complete! 2026-02-22T21:34:49.102 DEBUG:teuthology.orchestra.run.trial004:> sudo yum -y install ceph-test 2026-02-22T21:34:49.406 INFO:teuthology.orchestra.run.trial004.stdout:Last metadata expiration check: 0:00:14 ago on Sun 22 Feb 2026 09:34:35 PM UTC. 2026-02-22T21:34:49.488 INFO:teuthology.orchestra.run.trial004.stdout:Dependencies resolved. 2026-02-22T21:34:49.489 INFO:teuthology.orchestra.run.trial004.stdout:================================================================================ 2026-02-22T21:34:49.489 INFO:teuthology.orchestra.run.trial004.stdout: Package Arch Version Repository Size 2026-02-22T21:34:49.489 INFO:teuthology.orchestra.run.trial004.stdout:================================================================================ 2026-02-22T21:34:49.489 INFO:teuthology.orchestra.run.trial004.stdout:Installing: 2026-02-22T21:34:49.489 INFO:teuthology.orchestra.run.trial004.stdout: ceph-test x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 50 M 2026-02-22T21:34:49.489 INFO:teuthology.orchestra.run.trial004.stdout:Installing dependencies: 2026-02-22T21:34:49.489 INFO:teuthology.orchestra.run.trial004.stdout: libcephsqlite x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 163 k 2026-02-22T21:34:49.489 INFO:teuthology.orchestra.run.trial004.stdout: libxslt x86_64 1.1.34-12.el9 appstream 233 k 2026-02-22T21:34:49.489 INFO:teuthology.orchestra.run.trial004.stdout: socat x86_64 1.7.4.1-8.el9 appstream 303 k 2026-02-22T21:34:49.489 INFO:teuthology.orchestra.run.trial004.stdout: xmlstarlet x86_64 1.6.1-20.el9 appstream 64 k 2026-02-22T21:34:49.489 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:34:49.489 INFO:teuthology.orchestra.run.trial004.stdout:Transaction Summary 2026-02-22T21:34:49.489 INFO:teuthology.orchestra.run.trial004.stdout:================================================================================ 2026-02-22T21:34:49.489 INFO:teuthology.orchestra.run.trial004.stdout:Install 5 Packages 2026-02-22T21:34:49.489 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:34:49.490 INFO:teuthology.orchestra.run.trial004.stdout:Total download size: 50 M 2026-02-22T21:34:49.490 INFO:teuthology.orchestra.run.trial004.stdout:Installed size: 212 M 2026-02-22T21:34:49.490 INFO:teuthology.orchestra.run.trial004.stdout:Downloading Packages: 2026-02-22T21:34:49.902 INFO:teuthology.orchestra.run.trial004.stdout:(1/5): libxslt-1.1.34-12.el9.x86_64.rpm 3.7 MB/s | 233 kB 00:00 2026-02-22T21:34:49.920 INFO:teuthology.orchestra.run.trial004.stdout:(2/5): socat-1.7.4.1-8.el9.x86_64.rpm 16 MB/s | 303 kB 00:00 2026-02-22T21:34:49.929 INFO:teuthology.orchestra.run.trial004.stdout:(3/5): libcephsqlite-19.2.3-689.gf90f4eae.el9.x 1.8 MB/s | 163 kB 00:00 2026-02-22T21:34:49.930 INFO:teuthology.orchestra.run.trial004.stdout:(4/5): xmlstarlet-1.6.1-20.el9.x86_64.rpm 6.6 MB/s | 64 kB 00:00 2026-02-22T21:34:50.873 INFO:teuthology.orchestra.run.trial004.stdout:(5/5): ceph-test-19.2.3-689.gf90f4eae.el9.x86_6 48 MB/s | 50 MB 00:01 2026-02-22T21:34:50.875 INFO:teuthology.orchestra.run.trial004.stdout:-------------------------------------------------------------------------------- 2026-02-22T21:34:50.875 INFO:teuthology.orchestra.run.trial004.stdout:Total 36 MB/s | 50 MB 00:01 2026-02-22T21:34:50.900 INFO:teuthology.orchestra.run.trial004.stdout:Running transaction check 2026-02-22T21:34:50.909 INFO:teuthology.orchestra.run.trial004.stdout:Transaction check succeeded. 2026-02-22T21:34:50.910 INFO:teuthology.orchestra.run.trial004.stdout:Running transaction test 2026-02-22T21:34:51.096 INFO:teuthology.orchestra.run.trial004.stdout:Transaction test succeeded. 2026-02-22T21:34:51.096 INFO:teuthology.orchestra.run.trial004.stdout:Running transaction 2026-02-22T21:34:51.307 INFO:teuthology.orchestra.run.trial004.stdout: Preparing : 1/1 2026-02-22T21:34:51.313 INFO:teuthology.orchestra.run.trial004.stdout: Installing : socat-1.7.4.1-8.el9.x86_64 1/5 2026-02-22T21:34:51.318 INFO:teuthology.orchestra.run.trial004.stdout: Installing : libxslt-1.1.34-12.el9.x86_64 2/5 2026-02-22T21:34:51.325 INFO:teuthology.orchestra.run.trial004.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 3/5 2026-02-22T21:34:51.326 INFO:teuthology.orchestra.run.trial004.stdout: Installing : libcephsqlite-2:19.2.3-689.gf90f4eae.el9.x86_64 4/5 2026-02-22T21:34:52.551 INFO:teuthology.orchestra.run.trial026.stdout:CentOS Stream 9 - AppStream 1.0 MB/s | 27 MB 00:26 2026-02-22T21:34:53.644 INFO:teuthology.orchestra.run.trial004.stdout: Running scriptlet: libcephsqlite-2:19.2.3-689.gf90f4eae.el9.x86_64 4/5 2026-02-22T21:34:53.661 INFO:teuthology.orchestra.run.trial004.stdout: Installing : ceph-test-2:19.2.3-689.gf90f4eae.el9.x86_64 5/5 2026-02-22T21:34:53.894 INFO:teuthology.orchestra.run.trial004.stdout: Running scriptlet: ceph-test-2:19.2.3-689.gf90f4eae.el9.x86_64 5/5 2026-02-22T21:34:53.894 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : ceph-test-2:19.2.3-689.gf90f4eae.el9.x86_64 1/5 2026-02-22T21:34:53.894 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : libcephsqlite-2:19.2.3-689.gf90f4eae.el9.x86_64 2/5 2026-02-22T21:34:53.895 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : libxslt-1.1.34-12.el9.x86_64 3/5 2026-02-22T21:34:53.895 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : socat-1.7.4.1-8.el9.x86_64 4/5 2026-02-22T21:34:53.947 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 5/5 2026-02-22T21:34:53.947 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:34:53.947 INFO:teuthology.orchestra.run.trial004.stdout:Installed: 2026-02-22T21:34:53.947 INFO:teuthology.orchestra.run.trial004.stdout: ceph-test-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:34:53.947 INFO:teuthology.orchestra.run.trial004.stdout: libcephsqlite-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:34:53.947 INFO:teuthology.orchestra.run.trial004.stdout: libxslt-1.1.34-12.el9.x86_64 2026-02-22T21:34:53.947 INFO:teuthology.orchestra.run.trial004.stdout: socat-1.7.4.1-8.el9.x86_64 2026-02-22T21:34:53.947 INFO:teuthology.orchestra.run.trial004.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2026-02-22T21:34:53.947 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:34:53.947 INFO:teuthology.orchestra.run.trial004.stdout:Complete! 2026-02-22T21:34:53.980 DEBUG:teuthology.orchestra.run.trial004:> sudo yum -y install ceph 2026-02-22T21:34:54.285 INFO:teuthology.orchestra.run.trial004.stdout:Last metadata expiration check: 0:00:19 ago on Sun 22 Feb 2026 09:34:35 PM UTC. 2026-02-22T21:34:54.371 INFO:teuthology.orchestra.run.trial004.stdout:Dependencies resolved. 2026-02-22T21:34:54.374 INFO:teuthology.orchestra.run.trial004.stdout:====================================================================================== 2026-02-22T21:34:54.374 INFO:teuthology.orchestra.run.trial004.stdout: Package Arch Version Repository Size 2026-02-22T21:34:54.374 INFO:teuthology.orchestra.run.trial004.stdout:====================================================================================== 2026-02-22T21:34:54.374 INFO:teuthology.orchestra.run.trial004.stdout:Installing: 2026-02-22T21:34:54.374 INFO:teuthology.orchestra.run.trial004.stdout: ceph x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 6.5 k 2026-02-22T21:34:54.374 INFO:teuthology.orchestra.run.trial004.stdout:Installing dependencies: 2026-02-22T21:34:54.374 INFO:teuthology.orchestra.run.trial004.stdout: abseil-cpp x86_64 20211102.0-4.el9 epel 551 k 2026-02-22T21:34:54.374 INFO:teuthology.orchestra.run.trial004.stdout: ceph-grafana-dashboards noarch 2:19.2.3-689.gf90f4eae.el9 ceph-noarch 31 k 2026-02-22T21:34:54.374 INFO:teuthology.orchestra.run.trial004.stdout: ceph-mds x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 2.4 M 2026-02-22T21:34:54.374 INFO:teuthology.orchestra.run.trial004.stdout: ceph-mgr x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 1.1 M 2026-02-22T21:34:54.375 INFO:teuthology.orchestra.run.trial004.stdout: ceph-mgr-modules-core noarch 2:19.2.3-689.gf90f4eae.el9 ceph-noarch 253 k 2026-02-22T21:34:54.375 INFO:teuthology.orchestra.run.trial004.stdout: ceph-mon x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 4.7 M 2026-02-22T21:34:54.375 INFO:teuthology.orchestra.run.trial004.stdout: ceph-osd x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 17 M 2026-02-22T21:34:54.375 INFO:teuthology.orchestra.run.trial004.stdout: ceph-prometheus-alerts noarch 2:19.2.3-689.gf90f4eae.el9 ceph-noarch 17 k 2026-02-22T21:34:54.375 INFO:teuthology.orchestra.run.trial004.stdout: cephadm noarch 2:19.2.3-689.gf90f4eae.el9 ceph-noarch 769 k 2026-02-22T21:34:54.375 INFO:teuthology.orchestra.run.trial004.stdout: cryptsetup x86_64 2.8.1-3.el9 baseos 351 k 2026-02-22T21:34:54.375 INFO:teuthology.orchestra.run.trial004.stdout: flexiblas x86_64 3.0.4-9.el9 appstream 30 k 2026-02-22T21:34:54.375 INFO:teuthology.orchestra.run.trial004.stdout: flexiblas-netlib x86_64 3.0.4-9.el9 appstream 3.0 M 2026-02-22T21:34:54.375 INFO:teuthology.orchestra.run.trial004.stdout: flexiblas-openblas-openmp x86_64 3.0.4-9.el9 appstream 15 k 2026-02-22T21:34:54.375 INFO:teuthology.orchestra.run.trial004.stdout: grpc-data noarch 1.46.7-10.el9 epel 19 k 2026-02-22T21:34:54.375 INFO:teuthology.orchestra.run.trial004.stdout: ledmon-libs x86_64 1.1.0-3.el9 baseos 40 k 2026-02-22T21:34:54.376 INFO:teuthology.orchestra.run.trial004.stdout: libconfig x86_64 1.7.2-9.el9 baseos 72 k 2026-02-22T21:34:54.376 INFO:teuthology.orchestra.run.trial004.stdout: libgfortran x86_64 11.5.0-14.el9 baseos 794 k 2026-02-22T21:34:54.376 INFO:teuthology.orchestra.run.trial004.stdout: libquadmath x86_64 11.5.0-14.el9 baseos 184 k 2026-02-22T21:34:54.376 INFO:teuthology.orchestra.run.trial004.stdout: libstoragemgmt x86_64 1.10.1-1.el9 appstream 246 k 2026-02-22T21:34:54.376 INFO:teuthology.orchestra.run.trial004.stdout: lua x86_64 5.4.4-4.el9 appstream 188 k 2026-02-22T21:34:54.376 INFO:teuthology.orchestra.run.trial004.stdout: lua-devel x86_64 5.4.4-4.el9 crb 22 k 2026-02-22T21:34:54.376 INFO:teuthology.orchestra.run.trial004.stdout: luarocks noarch 3.9.2-5.el9 epel 151 k 2026-02-22T21:34:54.376 INFO:teuthology.orchestra.run.trial004.stdout: openblas x86_64 0.3.29-1.el9 appstream 42 k 2026-02-22T21:34:54.376 INFO:teuthology.orchestra.run.trial004.stdout: openblas-openmp x86_64 0.3.29-1.el9 appstream 5.3 M 2026-02-22T21:34:54.376 INFO:teuthology.orchestra.run.trial004.stdout: protobuf x86_64 3.14.0-17.el9 appstream 1.0 M 2026-02-22T21:34:54.376 INFO:teuthology.orchestra.run.trial004.stdout: protobuf-compiler x86_64 3.14.0-17.el9 crb 862 k 2026-02-22T21:34:54.376 INFO:teuthology.orchestra.run.trial004.stdout: python3-asyncssh noarch 2.13.2-5.el9 epel 548 k 2026-02-22T21:34:54.376 INFO:teuthology.orchestra.run.trial004.stdout: python3-autocommand noarch 2.2.2-8.el9 epel 29 k 2026-02-22T21:34:54.377 INFO:teuthology.orchestra.run.trial004.stdout: python3-babel noarch 2.9.1-2.el9 appstream 6.0 M 2026-02-22T21:34:54.377 INFO:teuthology.orchestra.run.trial004.stdout: python3-backports-tarfile noarch 1.2.0-1.el9 epel 60 k 2026-02-22T21:34:54.377 INFO:teuthology.orchestra.run.trial004.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2026-02-22T21:34:54.377 INFO:teuthology.orchestra.run.trial004.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2026-02-22T21:34:54.377 INFO:teuthology.orchestra.run.trial004.stdout: python3-certifi noarch 2023.05.07-4.el9 epel 14 k 2026-02-22T21:34:54.377 INFO:teuthology.orchestra.run.trial004.stdout: python3-cffi x86_64 1.14.5-5.el9 baseos 253 k 2026-02-22T21:34:54.377 INFO:teuthology.orchestra.run.trial004.stdout: python3-cheroot noarch 10.0.1-4.el9 epel 173 k 2026-02-22T21:34:54.377 INFO:teuthology.orchestra.run.trial004.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2026-02-22T21:34:54.377 INFO:teuthology.orchestra.run.trial004.stdout: python3-cryptography x86_64 36.0.1-5.el9 baseos 1.2 M 2026-02-22T21:34:54.377 INFO:teuthology.orchestra.run.trial004.stdout: python3-devel x86_64 3.9.25-3.el9 appstream 244 k 2026-02-22T21:34:54.377 INFO:teuthology.orchestra.run.trial004.stdout: python3-google-auth noarch 1:2.45.0-1.el9 epel 254 k 2026-02-22T21:34:54.377 INFO:teuthology.orchestra.run.trial004.stdout: python3-grpcio x86_64 1.46.7-10.el9 epel 2.0 M 2026-02-22T21:34:54.378 INFO:teuthology.orchestra.run.trial004.stdout: python3-grpcio-tools x86_64 1.46.7-10.el9 epel 144 k 2026-02-22T21:34:54.378 INFO:teuthology.orchestra.run.trial004.stdout: python3-isodate noarch 0.6.1-3.el9 epel 56 k 2026-02-22T21:34:54.378 INFO:teuthology.orchestra.run.trial004.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2026-02-22T21:34:54.378 INFO:teuthology.orchestra.run.trial004.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2026-02-22T21:34:54.378 INFO:teuthology.orchestra.run.trial004.stdout: python3-jaraco-collections noarch 3.0.0-8.el9 epel 23 k 2026-02-22T21:34:54.378 INFO:teuthology.orchestra.run.trial004.stdout: python3-jaraco-context noarch 6.0.1-3.el9 epel 20 k 2026-02-22T21:34:54.378 INFO:teuthology.orchestra.run.trial004.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2026-02-22T21:34:54.378 INFO:teuthology.orchestra.run.trial004.stdout: python3-jaraco-text noarch 4.0.0-2.el9 epel 26 k 2026-02-22T21:34:54.378 INFO:teuthology.orchestra.run.trial004.stdout: python3-jinja2 noarch 2.11.3-8.el9 appstream 249 k 2026-02-22T21:34:54.378 INFO:teuthology.orchestra.run.trial004.stdout: python3-kubernetes noarch 1:26.1.0-3.el9 epel 1.0 M 2026-02-22T21:34:54.378 INFO:teuthology.orchestra.run.trial004.stdout: python3-libstoragemgmt x86_64 1.10.1-1.el9 appstream 177 k 2026-02-22T21:34:54.378 INFO:teuthology.orchestra.run.trial004.stdout: python3-logutils noarch 0.3.5-21.el9 epel 46 k 2026-02-22T21:34:54.378 INFO:teuthology.orchestra.run.trial004.stdout: python3-lxml x86_64 4.6.5-3.el9 appstream 1.2 M 2026-02-22T21:34:54.379 INFO:teuthology.orchestra.run.trial004.stdout: python3-mako noarch 1.1.4-6.el9 appstream 172 k 2026-02-22T21:34:54.379 INFO:teuthology.orchestra.run.trial004.stdout: python3-markupsafe x86_64 1.1.1-12.el9 appstream 35 k 2026-02-22T21:34:54.379 INFO:teuthology.orchestra.run.trial004.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2026-02-22T21:34:54.379 INFO:teuthology.orchestra.run.trial004.stdout: python3-msgpack x86_64 1.0.3-2.el9 epel 86 k 2026-02-22T21:34:54.379 INFO:teuthology.orchestra.run.trial004.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2026-02-22T21:34:54.379 INFO:teuthology.orchestra.run.trial004.stdout: python3-numpy x86_64 1:1.23.5-2.el9 appstream 6.1 M 2026-02-22T21:34:54.379 INFO:teuthology.orchestra.run.trial004.stdout: python3-numpy-f2py x86_64 1:1.23.5-2.el9 appstream 442 k 2026-02-22T21:34:54.379 INFO:teuthology.orchestra.run.trial004.stdout: python3-packaging noarch 20.9-5.el9 appstream 77 k 2026-02-22T21:34:54.379 INFO:teuthology.orchestra.run.trial004.stdout: python3-pecan noarch 1.4.2-3.el9 epel 272 k 2026-02-22T21:34:54.379 INFO:teuthology.orchestra.run.trial004.stdout: python3-ply noarch 3.11-14.el9 baseos 106 k 2026-02-22T21:34:54.379 INFO:teuthology.orchestra.run.trial004.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2026-02-22T21:34:54.379 INFO:teuthology.orchestra.run.trial004.stdout: python3-protobuf noarch 3.14.0-17.el9 appstream 267 k 2026-02-22T21:34:54.379 INFO:teuthology.orchestra.run.trial004.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2026-02-22T21:34:54.380 INFO:teuthology.orchestra.run.trial004.stdout: python3-pyasn1 noarch 0.4.8-6.el9 appstream 159 k 2026-02-22T21:34:54.380 INFO:teuthology.orchestra.run.trial004.stdout: python3-pyasn1-modules noarch 0.4.8-6.el9 appstream 279 k 2026-02-22T21:34:54.380 INFO:teuthology.orchestra.run.trial004.stdout: python3-pycparser noarch 2.20-6.el9 baseos 135 k 2026-02-22T21:34:54.380 INFO:teuthology.orchestra.run.trial004.stdout: python3-pyparsing noarch 2.4.7-9.el9 baseos 150 k 2026-02-22T21:34:54.380 INFO:teuthology.orchestra.run.trial004.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2026-02-22T21:34:54.380 INFO:teuthology.orchestra.run.trial004.stdout: python3-requests noarch 2.25.1-10.el9 baseos 126 k 2026-02-22T21:34:54.380 INFO:teuthology.orchestra.run.trial004.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 appstream 54 k 2026-02-22T21:34:54.380 INFO:teuthology.orchestra.run.trial004.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2026-02-22T21:34:54.380 INFO:teuthology.orchestra.run.trial004.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2026-02-22T21:34:54.380 INFO:teuthology.orchestra.run.trial004.stdout: python3-scipy x86_64 1.9.3-2.el9 appstream 19 M 2026-02-22T21:34:54.380 INFO:teuthology.orchestra.run.trial004.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2026-02-22T21:34:54.380 INFO:teuthology.orchestra.run.trial004.stdout: python3-toml noarch 0.10.2-6.el9 appstream 42 k 2026-02-22T21:34:54.380 INFO:teuthology.orchestra.run.trial004.stdout: python3-typing-extensions noarch 4.15.0-1.el9 epel 86 k 2026-02-22T21:34:54.381 INFO:teuthology.orchestra.run.trial004.stdout: python3-urllib3 noarch 1.26.5-7.el9 baseos 218 k 2026-02-22T21:34:54.381 INFO:teuthology.orchestra.run.trial004.stdout: python3-webob noarch 1.8.8-2.el9 epel 230 k 2026-02-22T21:34:54.381 INFO:teuthology.orchestra.run.trial004.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2026-02-22T21:34:54.381 INFO:teuthology.orchestra.run.trial004.stdout: python3-werkzeug noarch 2.0.3-3.el9.1 epel 427 k 2026-02-22T21:34:54.381 INFO:teuthology.orchestra.run.trial004.stdout: python3-xmlsec x86_64 1.3.13-1.el9 epel 48 k 2026-02-22T21:34:54.381 INFO:teuthology.orchestra.run.trial004.stdout: python3-xmltodict noarch 0.12.0-15.el9 epel 22 k 2026-02-22T21:34:54.381 INFO:teuthology.orchestra.run.trial004.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2026-02-22T21:34:54.381 INFO:teuthology.orchestra.run.trial004.stdout: unzip x86_64 6.0-59.el9 baseos 182 k 2026-02-22T21:34:54.381 INFO:teuthology.orchestra.run.trial004.stdout: xmlsec1 x86_64 1.2.29-13.el9 appstream 189 k 2026-02-22T21:34:54.381 INFO:teuthology.orchestra.run.trial004.stdout: xmlsec1-openssl x86_64 1.2.29-13.el9 appstream 90 k 2026-02-22T21:34:54.381 INFO:teuthology.orchestra.run.trial004.stdout: zip x86_64 3.0-35.el9 baseos 266 k 2026-02-22T21:34:54.381 INFO:teuthology.orchestra.run.trial004.stdout:Installing weak dependencies: 2026-02-22T21:34:54.382 INFO:teuthology.orchestra.run.trial004.stdout: ceph-mgr-cephadm noarch 2:19.2.3-689.gf90f4eae.el9 ceph-noarch 150 k 2026-02-22T21:34:54.382 INFO:teuthology.orchestra.run.trial004.stdout: ceph-mgr-dashboard noarch 2:19.2.3-689.gf90f4eae.el9 ceph-noarch 3.8 M 2026-02-22T21:34:54.382 INFO:teuthology.orchestra.run.trial004.stdout: ceph-mgr-diskprediction-local noarch 2:19.2.3-689.gf90f4eae.el9 ceph-noarch 7.4 M 2026-02-22T21:34:54.382 INFO:teuthology.orchestra.run.trial004.stdout: ceph-mgr-k8sevents noarch 2:19.2.3-689.gf90f4eae.el9 ceph-noarch 22 k 2026-02-22T21:34:54.382 INFO:teuthology.orchestra.run.trial004.stdout: ceph-mgr-rook noarch 2:19.2.3-689.gf90f4eae.el9 ceph-noarch 49 k 2026-02-22T21:34:54.382 INFO:teuthology.orchestra.run.trial004.stdout: ceph-volume noarch 2:19.2.3-689.gf90f4eae.el9 ceph-noarch 299 k 2026-02-22T21:34:54.382 INFO:teuthology.orchestra.run.trial004.stdout: python3-influxdb noarch 5.3.1-1.el9 epel 139 k 2026-02-22T21:34:54.382 INFO:teuthology.orchestra.run.trial004.stdout: python3-saml noarch 1.16.0-1.el9 epel 125 k 2026-02-22T21:34:54.382 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:34:54.382 INFO:teuthology.orchestra.run.trial004.stdout:Transaction Summary 2026-02-22T21:34:54.382 INFO:teuthology.orchestra.run.trial004.stdout:====================================================================================== 2026-02-22T21:34:54.382 INFO:teuthology.orchestra.run.trial004.stdout:Install 99 Packages 2026-02-22T21:34:54.383 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:34:54.383 INFO:teuthology.orchestra.run.trial004.stdout:Total download size: 95 M 2026-02-22T21:34:54.383 INFO:teuthology.orchestra.run.trial004.stdout:Installed size: 520 M 2026-02-22T21:34:54.384 INFO:teuthology.orchestra.run.trial004.stdout:Downloading Packages: 2026-02-22T21:34:55.149 INFO:teuthology.orchestra.run.trial026.stdout:CentOS Stream 9 - CRB 32 MB/s | 7.8 MB 00:00 2026-02-22T21:34:55.356 INFO:teuthology.orchestra.run.trial004.stdout:(1/99): ceph-19.2.3-689.gf90f4eae.el9.x86_64.rp 34 kB/s | 6.5 kB 00:00 2026-02-22T21:34:55.751 INFO:teuthology.orchestra.run.trial004.stdout:(2/99): ceph-mgr-19.2.3-689.gf90f4eae.el9.x86_6 1.8 MB/s | 1.1 MB 00:00 2026-02-22T21:34:55.967 INFO:teuthology.orchestra.run.trial026.stdout:CentOS Stream 9 - Extras packages 195 kB/s | 20 kB 00:00 2026-02-22T21:34:55.994 INFO:teuthology.orchestra.run.trial004.stdout:(3/99): ceph-mds-19.2.3-689.gf90f4eae.el9.x86_6 2.9 MB/s | 2.4 MB 00:00 2026-02-22T21:34:56.054 INFO:teuthology.orchestra.run.trial004.stdout:(4/99): ceph-grafana-dashboards-19.2.3-689.gf90 521 kB/s | 31 kB 00:00 2026-02-22T21:34:56.117 INFO:teuthology.orchestra.run.trial004.stdout:(5/99): ceph-mgr-cephadm-19.2.3-689.gf90f4eae.e 2.3 MB/s | 150 kB 00:00 2026-02-22T21:34:56.443 INFO:teuthology.orchestra.run.trial004.stdout:(6/99): ceph-mon-19.2.3-689.gf90f4eae.el9.x86_6 4.4 MB/s | 4.7 MB 00:01 2026-02-22T21:34:56.620 INFO:teuthology.orchestra.run.trial026.stdout:Extra Packages for Enterprise Linux 34 MB/s | 20 MB 00:00 2026-02-22T21:34:56.777 INFO:teuthology.orchestra.run.trial004.stdout:(7/99): ceph-mgr-dashboard-19.2.3-689.gf90f4eae 5.8 MB/s | 3.8 MB 00:00 2026-02-22T21:34:56.832 INFO:teuthology.orchestra.run.trial004.stdout:(8/99): ceph-mgr-k8sevents-19.2.3-689.gf90f4eae 404 kB/s | 22 kB 00:00 2026-02-22T21:34:56.892 INFO:teuthology.orchestra.run.trial004.stdout:(9/99): ceph-mgr-modules-core-19.2.3-689.gf90f4 4.1 MB/s | 253 kB 00:00 2026-02-22T21:34:56.948 INFO:teuthology.orchestra.run.trial004.stdout:(10/99): ceph-mgr-rook-19.2.3-689.gf90f4eae.el9 883 kB/s | 49 kB 00:00 2026-02-22T21:34:57.003 INFO:teuthology.orchestra.run.trial004.stdout:(11/99): ceph-prometheus-alerts-19.2.3-689.gf90 301 kB/s | 17 kB 00:00 2026-02-22T21:34:57.064 INFO:teuthology.orchestra.run.trial004.stdout:(12/99): ceph-volume-19.2.3-689.gf90f4eae.el9.n 4.8 MB/s | 299 kB 00:00 2026-02-22T21:34:57.249 INFO:teuthology.orchestra.run.trial004.stdout:(13/99): cephadm-19.2.3-689.gf90f4eae.el9.noarc 4.1 MB/s | 769 kB 00:00 2026-02-22T21:34:57.301 INFO:teuthology.orchestra.run.trial004.stdout:(14/99): cryptsetup-2.8.1-3.el9.x86_64.rpm 6.7 MB/s | 351 kB 00:00 2026-02-22T21:34:57.310 INFO:teuthology.orchestra.run.trial004.stdout:(15/99): ledmon-libs-1.1.0-3.el9.x86_64.rpm 4.4 MB/s | 40 kB 00:00 2026-02-22T21:34:57.320 INFO:teuthology.orchestra.run.trial004.stdout:(16/99): libconfig-1.7.2-9.el9.x86_64.rpm 7.6 MB/s | 72 kB 00:00 2026-02-22T21:34:57.340 INFO:teuthology.orchestra.run.trial004.stdout:(17/99): libgfortran-11.5.0-14.el9.x86_64.rpm 38 MB/s | 794 kB 00:00 2026-02-22T21:34:57.351 INFO:teuthology.orchestra.run.trial004.stdout:(18/99): libquadmath-11.5.0-14.el9.x86_64.rpm 16 MB/s | 184 kB 00:00 2026-02-22T21:34:57.363 INFO:teuthology.orchestra.run.trial004.stdout:(19/99): python3-cffi-1.14.5-5.el9.x86_64.rpm 22 MB/s | 253 kB 00:00 2026-02-22T21:34:57.396 INFO:teuthology.orchestra.run.trial004.stdout:(20/99): python3-cryptography-36.0.1-5.el9.x86_ 38 MB/s | 1.2 MB 00:00 2026-02-22T21:34:57.406 INFO:teuthology.orchestra.run.trial004.stdout:(21/99): python3-ply-3.11-14.el9.noarch.rpm 10 MB/s | 106 kB 00:00 2026-02-22T21:34:57.417 INFO:teuthology.orchestra.run.trial004.stdout:(22/99): python3-pycparser-2.20-6.el9.noarch.rp 12 MB/s | 135 kB 00:00 2026-02-22T21:34:57.428 INFO:teuthology.orchestra.run.trial004.stdout:(23/99): python3-pyparsing-2.4.7-9.el9.noarch.r 14 MB/s | 150 kB 00:00 2026-02-22T21:34:57.439 INFO:teuthology.orchestra.run.trial004.stdout:(24/99): python3-requests-2.25.1-10.el9.noarch. 11 MB/s | 126 kB 00:00 2026-02-22T21:34:57.450 INFO:teuthology.orchestra.run.trial004.stdout:(25/99): python3-urllib3-1.26.5-7.el9.noarch.rp 19 MB/s | 218 kB 00:00 2026-02-22T21:34:57.462 INFO:teuthology.orchestra.run.trial004.stdout:(26/99): unzip-6.0-59.el9.x86_64.rpm 16 MB/s | 182 kB 00:00 2026-02-22T21:34:57.473 INFO:teuthology.orchestra.run.trial004.stdout:(27/99): zip-3.0-35.el9.x86_64.rpm 24 MB/s | 266 kB 00:00 2026-02-22T21:34:57.558 INFO:teuthology.orchestra.run.trial004.stdout:(28/99): flexiblas-3.0.4-9.el9.x86_64.rpm 346 kB/s | 30 kB 00:00 2026-02-22T21:34:57.658 INFO:teuthology.orchestra.run.trial004.stdout:(29/99): ceph-mgr-diskprediction-local-19.2.3-6 6.1 MB/s | 7.4 MB 00:01 2026-02-22T21:34:57.710 INFO:teuthology.orchestra.run.trial004.stdout:(30/99): flexiblas-netlib-3.0.4-9.el9.x86_64.rp 20 MB/s | 3.0 MB 00:00 2026-02-22T21:34:57.714 INFO:teuthology.orchestra.run.trial004.stdout:(31/99): flexiblas-openblas-openmp-3.0.4-9.el9. 266 kB/s | 15 kB 00:00 2026-02-22T21:34:57.734 INFO:teuthology.orchestra.run.trial004.stdout:(32/99): libstoragemgmt-1.10.1-1.el9.x86_64.rpm 9.9 MB/s | 246 kB 00:00 2026-02-22T21:34:57.756 INFO:teuthology.orchestra.run.trial004.stdout:(33/99): openblas-0.3.29-1.el9.x86_64.rpm 1.8 MB/s | 42 kB 00:00 2026-02-22T21:34:57.771 INFO:teuthology.orchestra.run.trial004.stdout:(34/99): lua-5.4.4-4.el9.x86_64.rpm 3.2 MB/s | 188 kB 00:00 2026-02-22T21:34:57.810 INFO:teuthology.orchestra.run.trial004.stdout:(35/99): openblas-openmp-0.3.29-1.el9.x86_64.rp 98 MB/s | 5.3 MB 00:00 2026-02-22T21:34:57.846 INFO:teuthology.orchestra.run.trial004.stdout:(36/99): protobuf-3.14.0-17.el9.x86_64.rpm 14 MB/s | 1.0 MB 00:00 2026-02-22T21:34:57.869 INFO:teuthology.orchestra.run.trial004.stdout:(37/99): python3-devel-3.9.25-3.el9.x86_64.rpm 10 MB/s | 244 kB 00:00 2026-02-22T21:34:57.898 INFO:teuthology.orchestra.run.trial004.stdout:(38/99): python3-babel-2.9.1-2.el9.noarch.rpm 68 MB/s | 6.0 MB 00:00 2026-02-22T21:34:57.898 INFO:teuthology.orchestra.run.trial004.stdout:(39/99): python3-jinja2-2.11.3-8.el9.noarch.rpm 8.5 MB/s | 249 kB 00:00 2026-02-22T21:34:57.922 INFO:teuthology.orchestra.run.trial004.stdout:(40/99): python3-libstoragemgmt-1.10.1-1.el9.x8 7.0 MB/s | 177 kB 00:00 2026-02-22T21:34:57.947 INFO:teuthology.orchestra.run.trial004.stdout:(41/99): python3-lxml-4.6.5-3.el9.x86_64.rpm 25 MB/s | 1.2 MB 00:00 2026-02-22T21:34:57.947 INFO:teuthology.orchestra.run.trial004.stdout:(42/99): python3-mako-1.1.4-6.el9.noarch.rpm 6.7 MB/s | 172 kB 00:00 2026-02-22T21:34:57.967 INFO:teuthology.orchestra.run.trial004.stdout:(43/99): python3-markupsafe-1.1.1-12.el9.x86_64 1.7 MB/s | 35 kB 00:00 2026-02-22T21:34:57.992 INFO:teuthology.orchestra.run.trial004.stdout:(44/99): python3-numpy-f2py-1.23.5-2.el9.x86_64 17 MB/s | 442 kB 00:00 2026-02-22T21:34:58.013 INFO:teuthology.orchestra.run.trial004.stdout:(45/99): python3-packaging-20.9-5.el9.noarch.rp 3.6 MB/s | 77 kB 00:00 2026-02-22T21:34:58.025 INFO:teuthology.orchestra.run.trial004.stdout:(46/99): python3-numpy-1.23.5-2.el9.x86_64.rpm 79 MB/s | 6.1 MB 00:00 2026-02-22T21:34:58.035 INFO:teuthology.orchestra.run.trial004.stdout:(47/99): python3-protobuf-3.14.0-17.el9.noarch. 12 MB/s | 267 kB 00:00 2026-02-22T21:34:58.050 INFO:teuthology.orchestra.run.trial004.stdout:(48/99): python3-pyasn1-0.4.8-6.el9.noarch.rpm 6.4 MB/s | 159 kB 00:00 2026-02-22T21:34:58.057 INFO:teuthology.orchestra.run.trial004.stdout:(49/99): python3-pyasn1-modules-0.4.8-6.el9.noa 12 MB/s | 279 kB 00:00 2026-02-22T21:34:58.072 INFO:teuthology.orchestra.run.trial004.stdout:(50/99): python3-requests-oauthlib-1.3.0-12.el9 2.3 MB/s | 54 kB 00:00 2026-02-22T21:34:58.095 INFO:teuthology.orchestra.run.trial004.stdout:(51/99): python3-toml-0.10.2-6.el9.noarch.rpm 1.8 MB/s | 42 kB 00:00 2026-02-22T21:34:58.122 INFO:teuthology.orchestra.run.trial004.stdout:(52/99): xmlsec1-1.2.29-13.el9.x86_64.rpm 6.8 MB/s | 189 kB 00:00 2026-02-22T21:34:58.145 INFO:teuthology.orchestra.run.trial004.stdout:(53/99): xmlsec1-openssl-1.2.29-13.el9.x86_64.r 3.9 MB/s | 90 kB 00:00 2026-02-22T21:34:58.479 INFO:teuthology.orchestra.run.trial004.stdout:(54/99): python3-scipy-1.9.3-2.el9.x86_64.rpm 46 MB/s | 19 MB 00:00 2026-02-22T21:34:58.517 INFO:teuthology.orchestra.run.trial004.stdout:(55/99): ceph-osd-19.2.3-689.gf90f4eae.el9.x86_ 6.2 MB/s | 17 MB 00:02 2026-02-22T21:34:58.648 INFO:teuthology.orchestra.run.trial004.stdout:(56/99): abseil-cpp-20211102.0-4.el9.x86_64.rpm 4.1 MB/s | 551 kB 00:00 2026-02-22T21:34:58.665 INFO:teuthology.orchestra.run.trial004.stdout:(57/99): grpc-data-1.46.7-10.el9.noarch.rpm 1.1 MB/s | 19 kB 00:00 2026-02-22T21:34:58.684 INFO:teuthology.orchestra.run.trial004.stdout:(58/99): luarocks-3.9.2-5.el9.noarch.rpm 7.8 MB/s | 151 kB 00:00 2026-02-22T21:34:58.714 INFO:teuthology.orchestra.run.trial004.stdout:(59/99): python3-asyncssh-2.13.2-5.el9.noarch.r 18 MB/s | 548 kB 00:00 2026-02-22T21:34:58.717 INFO:teuthology.orchestra.run.trial004.stdout:(60/99): lua-devel-5.4.4-4.el9.x86_64.rpm 39 kB/s | 22 kB 00:00 2026-02-22T21:34:58.732 INFO:teuthology.orchestra.run.trial004.stdout:(61/99): python3-autocommand-2.2.2-8.el9.noarch 1.6 MB/s | 29 kB 00:00 2026-02-22T21:34:58.750 INFO:teuthology.orchestra.run.trial004.stdout:(62/99): python3-bcrypt-3.2.2-1.el9.x86_64.rpm 2.3 MB/s | 43 kB 00:00 2026-02-22T21:34:58.768 INFO:teuthology.orchestra.run.trial004.stdout:(63/99): python3-cachetools-4.2.4-1.el9.noarch. 1.8 MB/s | 32 kB 00:00 2026-02-22T21:34:58.785 INFO:teuthology.orchestra.run.trial004.stdout:(64/99): python3-backports-tarfile-1.2.0-1.el9. 895 kB/s | 60 kB 00:00 2026-02-22T21:34:58.786 INFO:teuthology.orchestra.run.trial004.stdout:(65/99): python3-certifi-2023.05.07-4.el9.noarc 795 kB/s | 14 kB 00:00 2026-02-22T21:34:58.809 INFO:teuthology.orchestra.run.trial004.stdout:(66/99): python3-cherrypy-18.6.1-2.el9.noarch.r 15 MB/s | 358 kB 00:00 2026-02-22T21:34:58.830 INFO:teuthology.orchestra.run.trial004.stdout:(67/99): python3-google-auth-2.45.0-1.el9.noarc 12 MB/s | 254 kB 00:00 2026-02-22T21:34:58.831 INFO:teuthology.orchestra.run.trial004.stdout:(68/99): python3-cheroot-10.0.1-4.el9.noarch.rp 3.6 MB/s | 173 kB 00:00 2026-02-22T21:34:58.856 INFO:teuthology.orchestra.run.trial004.stdout:(69/99): python3-grpcio-tools-1.46.7-10.el9.x86 5.8 MB/s | 144 kB 00:00 2026-02-22T21:34:58.879 INFO:teuthology.orchestra.run.trial004.stdout:(70/99): python3-influxdb-5.3.1-1.el9.noarch.rp 5.8 MB/s | 139 kB 00:00 2026-02-22T21:34:58.890 INFO:teuthology.orchestra.run.trial004.stdout:(71/99): python3-grpcio-1.46.7-10.el9.x86_64.rp 34 MB/s | 2.0 MB 00:00 2026-02-22T21:34:58.898 INFO:teuthology.orchestra.run.trial004.stdout:(72/99): python3-isodate-0.6.1-3.el9.noarch.rpm 2.9 MB/s | 56 kB 00:00 2026-02-22T21:34:58.907 INFO:teuthology.orchestra.run.trial004.stdout:(73/99): python3-jaraco-8.2.1-3.el9.noarch.rpm 598 kB/s | 11 kB 00:00 2026-02-22T21:34:58.915 INFO:teuthology.orchestra.run.trial004.stdout:(74/99): python3-jaraco-classes-3.2.1-5.el9.noa 1.0 MB/s | 18 kB 00:00 2026-02-22T21:34:58.926 INFO:teuthology.orchestra.run.trial004.stdout:(75/99): python3-jaraco-collections-3.0.0-8.el9 1.2 MB/s | 23 kB 00:00 2026-02-22T21:34:58.931 INFO:teuthology.orchestra.run.trial004.stdout:(76/99): python3-jaraco-context-6.0.1-3.el9.noa 1.2 MB/s | 20 kB 00:00 2026-02-22T21:34:58.944 INFO:teuthology.orchestra.run.trial004.stdout:(77/99): python3-jaraco-functools-3.5.0-2.el9.n 1.1 MB/s | 19 kB 00:00 2026-02-22T21:34:58.949 INFO:teuthology.orchestra.run.trial004.stdout:(78/99): python3-jaraco-text-4.0.0-2.el9.noarch 1.5 MB/s | 26 kB 00:00 2026-02-22T21:34:58.966 INFO:teuthology.orchestra.run.trial004.stdout:(79/99): python3-logutils-0.3.5-21.el9.noarch.r 2.6 MB/s | 46 kB 00:00 2026-02-22T21:34:58.968 INFO:teuthology.orchestra.run.trial004.stdout:(80/99): python3-kubernetes-26.1.0-3.el9.noarch 42 MB/s | 1.0 MB 00:00 2026-02-22T21:34:58.986 INFO:teuthology.orchestra.run.trial004.stdout:(81/99): python3-more-itertools-8.12.0-2.el9.no 3.9 MB/s | 79 kB 00:00 2026-02-22T21:34:58.987 INFO:teuthology.orchestra.run.trial004.stdout:(82/99): python3-msgpack-1.0.3-2.el9.x86_64.rpm 4.5 MB/s | 86 kB 00:00 2026-02-22T21:34:59.005 INFO:teuthology.orchestra.run.trial004.stdout:(83/99): python3-natsort-7.1.1-5.el9.noarch.rpm 3.0 MB/s | 58 kB 00:00 2026-02-22T21:34:59.007 INFO:teuthology.orchestra.run.trial004.stdout:(84/99): python3-pecan-1.4.2-3.el9.noarch.rpm 14 MB/s | 272 kB 00:00 2026-02-22T21:34:59.021 INFO:teuthology.orchestra.run.trial004.stdout:(85/99): python3-portend-3.1.0-2.el9.noarch.rpm 986 kB/s | 16 kB 00:00 2026-02-22T21:34:59.026 INFO:teuthology.orchestra.run.trial004.stdout:(86/99): python3-pyOpenSSL-21.0.0-1.el9.noarch. 4.6 MB/s | 90 kB 00:00 2026-02-22T21:34:59.039 INFO:teuthology.orchestra.run.trial004.stdout:(87/99): python3-repoze-lru-0.7-16.el9.noarch.r 1.8 MB/s | 31 kB 00:00 2026-02-22T21:34:59.045 INFO:teuthology.orchestra.run.trial004.stdout:(88/99): python3-routes-2.5.1-5.el9.noarch.rpm 9.8 MB/s | 188 kB 00:00 2026-02-22T21:34:59.057 INFO:teuthology.orchestra.run.trial004.stdout:(89/99): python3-rsa-4.9-2.el9.noarch.rpm 3.1 MB/s | 59 kB 00:00 2026-02-22T21:34:59.064 INFO:teuthology.orchestra.run.trial004.stdout:(90/99): python3-saml-1.16.0-1.el9.noarch.rpm 6.3 MB/s | 125 kB 00:00 2026-02-22T21:34:59.075 INFO:teuthology.orchestra.run.trial004.stdout:(91/99): python3-tempora-5.0.0-2.el9.noarch.rpm 2.0 MB/s | 36 kB 00:00 2026-02-22T21:34:59.082 INFO:teuthology.orchestra.run.trial004.stdout:(92/99): python3-typing-extensions-4.15.0-1.el9 4.7 MB/s | 86 kB 00:00 2026-02-22T21:34:59.102 INFO:teuthology.orchestra.run.trial004.stdout:(93/99): python3-websocket-client-1.2.3-2.el9.n 4.6 MB/s | 90 kB 00:00 2026-02-22T21:34:59.104 INFO:teuthology.orchestra.run.trial004.stdout:(94/99): python3-webob-1.8.8-2.el9.noarch.rpm 7.9 MB/s | 230 kB 00:00 2026-02-22T21:34:59.122 INFO:teuthology.orchestra.run.trial004.stdout:(95/99): python3-werkzeug-2.0.3-3.el9.1.noarch. 21 MB/s | 427 kB 00:00 2026-02-22T21:34:59.123 INFO:teuthology.orchestra.run.trial004.stdout:(96/99): python3-xmlsec-1.3.13-1.el9.x86_64.rpm 2.6 MB/s | 48 kB 00:00 2026-02-22T21:34:59.139 INFO:teuthology.orchestra.run.trial004.stdout:(97/99): python3-zc-lockfile-2.0-10.el9.noarch. 1.2 MB/s | 20 kB 00:00 2026-02-22T21:34:59.140 INFO:teuthology.orchestra.run.trial004.stdout:(98/99): python3-xmltodict-0.12.0-15.el9.noarch 1.2 MB/s | 22 kB 00:00 2026-02-22T21:34:59.746 INFO:teuthology.orchestra.run.trial026.stdout:lab-extras 2.8 MB/s | 50 kB 00:00 2026-02-22T21:34:59.970 INFO:teuthology.orchestra.run.trial004.stdout:(99/99): protobuf-compiler-3.14.0-17.el9.x86_64 578 kB/s | 862 kB 00:01 2026-02-22T21:34:59.975 INFO:teuthology.orchestra.run.trial004.stdout:-------------------------------------------------------------------------------- 2026-02-22T21:34:59.975 INFO:teuthology.orchestra.run.trial004.stdout:Total 17 MB/s | 95 MB 00:05 2026-02-22T21:35:00.409 INFO:teuthology.orchestra.run.trial004.stdout:Running transaction check 2026-02-22T21:35:00.447 INFO:teuthology.orchestra.run.trial004.stdout:Transaction check succeeded. 2026-02-22T21:35:00.447 INFO:teuthology.orchestra.run.trial004.stdout:Running transaction test 2026-02-22T21:35:00.863 INFO:teuthology.orchestra.run.trial004.stdout:Transaction test succeeded. 2026-02-22T21:35:00.863 INFO:teuthology.orchestra.run.trial004.stdout:Running transaction 2026-02-22T21:35:00.953 INFO:teuthology.orchestra.run.trial026.stdout:Dependencies resolved. 2026-02-22T21:35:00.954 INFO:teuthology.orchestra.run.trial026.stdout:================================================================================ 2026-02-22T21:35:00.954 INFO:teuthology.orchestra.run.trial026.stdout: Package Arch Version Repository Size 2026-02-22T21:35:00.954 INFO:teuthology.orchestra.run.trial026.stdout:================================================================================ 2026-02-22T21:35:00.954 INFO:teuthology.orchestra.run.trial026.stdout:Installing: 2026-02-22T21:35:00.954 INFO:teuthology.orchestra.run.trial026.stdout: ceph-radosgw x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 11 M 2026-02-22T21:35:00.954 INFO:teuthology.orchestra.run.trial026.stdout:Upgrading: 2026-02-22T21:35:00.955 INFO:teuthology.orchestra.run.trial026.stdout: librados2 x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 3.4 M 2026-02-22T21:35:00.955 INFO:teuthology.orchestra.run.trial026.stdout: librbd1 x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 3.2 M 2026-02-22T21:35:00.955 INFO:teuthology.orchestra.run.trial026.stdout:Installing dependencies: 2026-02-22T21:35:00.955 INFO:teuthology.orchestra.run.trial026.stdout: boost-program-options x86_64 1.75.0-13.el9 appstream 104 k 2026-02-22T21:35:00.955 INFO:teuthology.orchestra.run.trial026.stdout: ceph-base x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 5.5 M 2026-02-22T21:35:00.955 INFO:teuthology.orchestra.run.trial026.stdout: ceph-common x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 22 M 2026-02-22T21:35:00.956 INFO:teuthology.orchestra.run.trial026.stdout: ceph-selinux x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 25 k 2026-02-22T21:35:00.956 INFO:teuthology.orchestra.run.trial026.stdout: gperftools-libs x86_64 2.9.1-3.el9 epel 308 k 2026-02-22T21:35:00.956 INFO:teuthology.orchestra.run.trial026.stdout: libarrow x86_64 9.0.0-15.el9 epel 4.4 M 2026-02-22T21:35:00.956 INFO:teuthology.orchestra.run.trial026.stdout: libarrow-doc noarch 9.0.0-15.el9 epel 25 k 2026-02-22T21:35:00.956 INFO:teuthology.orchestra.run.trial026.stdout: libcephfs2 x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 999 k 2026-02-22T21:35:00.956 INFO:teuthology.orchestra.run.trial026.stdout: libnbd x86_64 1.20.3-4.el9 appstream 164 k 2026-02-22T21:35:00.956 INFO:teuthology.orchestra.run.trial026.stdout: liboath x86_64 2.6.12-1.el9 epel 49 k 2026-02-22T21:35:00.957 INFO:teuthology.orchestra.run.trial026.stdout: libpmemobj x86_64 1.12.1-1.el9 appstream 160 k 2026-02-22T21:35:00.957 INFO:teuthology.orchestra.run.trial026.stdout: librabbitmq x86_64 0.11.0-7.el9 appstream 45 k 2026-02-22T21:35:00.957 INFO:teuthology.orchestra.run.trial026.stdout: libradosstriper1 x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 503 k 2026-02-22T21:35:00.957 INFO:teuthology.orchestra.run.trial026.stdout: librdkafka x86_64 1.6.1-102.el9 appstream 662 k 2026-02-22T21:35:00.957 INFO:teuthology.orchestra.run.trial026.stdout: librgw2 x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 5.4 M 2026-02-22T21:35:00.957 INFO:teuthology.orchestra.run.trial026.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2026-02-22T21:35:00.958 INFO:teuthology.orchestra.run.trial026.stdout: lttng-ust x86_64 2.12.0-6.el9 appstream 292 k 2026-02-22T21:35:00.958 INFO:teuthology.orchestra.run.trial026.stdout: mailcap noarch 2.1.49-5.el9 baseos 33 k 2026-02-22T21:35:00.958 INFO:teuthology.orchestra.run.trial026.stdout: parquet-libs x86_64 9.0.0-15.el9 epel 838 k 2026-02-22T21:35:00.958 INFO:teuthology.orchestra.run.trial026.stdout: pciutils x86_64 3.7.0-7.el9 baseos 93 k 2026-02-22T21:35:00.958 INFO:teuthology.orchestra.run.trial026.stdout: python3-ceph-argparse x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 45 k 2026-02-22T21:35:00.958 INFO:teuthology.orchestra.run.trial026.stdout: python3-ceph-common x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 142 k 2026-02-22T21:35:00.958 INFO:teuthology.orchestra.run.trial026.stdout: python3-cephfs x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 165 k 2026-02-22T21:35:00.958 INFO:teuthology.orchestra.run.trial026.stdout: python3-rados x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 322 k 2026-02-22T21:35:00.959 INFO:teuthology.orchestra.run.trial026.stdout: python3-rbd x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 303 k 2026-02-22T21:35:00.959 INFO:teuthology.orchestra.run.trial026.stdout: python3-rgw x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 100 k 2026-02-22T21:35:00.959 INFO:teuthology.orchestra.run.trial026.stdout: qatlib x86_64 25.08.0-2.el9 appstream 240 k 2026-02-22T21:35:00.959 INFO:teuthology.orchestra.run.trial026.stdout: qatzip-libs x86_64 1.3.1-1.el9 appstream 66 k 2026-02-22T21:35:00.959 INFO:teuthology.orchestra.run.trial026.stdout: re2 x86_64 1:20211101-20.el9 epel 191 k 2026-02-22T21:35:00.959 INFO:teuthology.orchestra.run.trial026.stdout: thrift x86_64 0.15.0-4.el9 epel 1.6 M 2026-02-22T21:35:00.959 INFO:teuthology.orchestra.run.trial026.stdout:Installing weak dependencies: 2026-02-22T21:35:00.960 INFO:teuthology.orchestra.run.trial026.stdout: nvme-cli x86_64 2.16-1.el9 baseos 1.2 M 2026-02-22T21:35:00.960 INFO:teuthology.orchestra.run.trial026.stdout: qatlib-service x86_64 25.08.0-2.el9 appstream 37 k 2026-02-22T21:35:00.960 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:35:00.960 INFO:teuthology.orchestra.run.trial026.stdout:Transaction Summary 2026-02-22T21:35:00.960 INFO:teuthology.orchestra.run.trial026.stdout:================================================================================ 2026-02-22T21:35:00.960 INFO:teuthology.orchestra.run.trial026.stdout:Install 33 Packages 2026-02-22T21:35:00.961 INFO:teuthology.orchestra.run.trial026.stdout:Upgrade 2 Packages 2026-02-22T21:35:00.961 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:35:00.962 INFO:teuthology.orchestra.run.trial026.stdout:Total download size: 63 M 2026-02-22T21:35:00.962 INFO:teuthology.orchestra.run.trial026.stdout:Downloading Packages: 2026-02-22T21:35:01.328 INFO:teuthology.orchestra.run.trial004.stdout: Preparing : 1/1 2026-02-22T21:35:01.330 INFO:teuthology.orchestra.run.trial004.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 1/99 2026-02-22T21:35:01.362 INFO:teuthology.orchestra.run.trial004.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 2/99 2026-02-22T21:35:01.367 INFO:teuthology.orchestra.run.trial004.stdout: Installing : python3-werkzeug-2.0.3-3.el9.1.noarch 3/99 2026-02-22T21:35:01.388 INFO:teuthology.orchestra.run.trial004.stdout: Installing : xmlsec1-1.2.29-13.el9.x86_64 4/99 2026-02-22T21:35:01.394 INFO:teuthology.orchestra.run.trial004.stdout: Installing : python3-pyasn1-0.4.8-6.el9.noarch 5/99 2026-02-22T21:35:01.428 INFO:teuthology.orchestra.run.trial004.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 6/99 2026-02-22T21:35:01.437 INFO:teuthology.orchestra.run.trial004.stdout: Installing : python3-lxml-4.6.5-3.el9.x86_64 7/99 2026-02-22T21:35:01.442 INFO:teuthology.orchestra.run.trial004.stdout: Installing : protobuf-3.14.0-17.el9.x86_64 8/99 2026-02-22T21:35:01.445 INFO:teuthology.orchestra.run.trial004.stdout: Installing : lua-5.4.4-4.el9.x86_64 9/99 2026-02-22T21:35:01.449 INFO:teuthology.orchestra.run.trial004.stdout: Installing : flexiblas-3.0.4-9.el9.x86_64 10/99 2026-02-22T21:35:01.473 INFO:teuthology.orchestra.run.trial004.stdout: Installing : unzip-6.0-59.el9.x86_64 11/99 2026-02-22T21:35:01.485 INFO:teuthology.orchestra.run.trial004.stdout: Installing : python3-urllib3-1.26.5-7.el9.noarch 12/99 2026-02-22T21:35:01.488 INFO:teuthology.orchestra.run.trial004.stdout: Installing : python3-requests-2.25.1-10.el9.noarch 13/99 2026-02-22T21:35:01.494 INFO:teuthology.orchestra.run.trial004.stdout: Installing : libquadmath-11.5.0-14.el9.x86_64 14/99 2026-02-22T21:35:01.496 INFO:teuthology.orchestra.run.trial004.stdout: Installing : libgfortran-11.5.0-14.el9.x86_64 15/99 2026-02-22T21:35:01.507 INFO:teuthology.orchestra.run.trial004.stdout: Installing : ledmon-libs-1.1.0-3.el9.x86_64 16/99 2026-02-22T21:35:01.514 INFO:teuthology.orchestra.run.trial004.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 17/99 2026-02-22T21:35:01.538 INFO:teuthology.orchestra.run.trial004.stdout: Installing : zip-3.0-35.el9.x86_64 18/99 2026-02-22T21:35:01.541 INFO:teuthology.orchestra.run.trial004.stdout: Installing : luarocks-3.9.2-5.el9.noarch 19/99 2026-02-22T21:35:01.548 INFO:teuthology.orchestra.run.trial004.stdout: Installing : lua-devel-5.4.4-4.el9.x86_64 20/99 2026-02-22T21:35:01.573 INFO:teuthology.orchestra.run.trial004.stdout: Installing : protobuf-compiler-3.14.0-17.el9.x86_64 21/99 2026-02-22T21:35:01.624 INFO:teuthology.orchestra.run.trial004.stdout: Installing : python3-mako-1.1.4-6.el9.noarch 22/99 2026-02-22T21:35:01.637 INFO:teuthology.orchestra.run.trial004.stdout: Installing : python3-pyasn1-modules-0.4.8-6.el9.noarch 23/99 2026-02-22T21:35:01.639 INFO:teuthology.orchestra.run.trial004.stdout: Installing : python3-rsa-4.9-2.el9.noarch 24/99 2026-02-22T21:35:01.644 INFO:teuthology.orchestra.run.trial004.stdout: Installing : xmlsec1-openssl-1.2.29-13.el9.x86_64 25/99 2026-02-22T21:35:01.649 INFO:teuthology.orchestra.run.trial004.stdout: Installing : python3-xmlsec-1.3.13-1.el9.x86_64 26/99 2026-02-22T21:35:01.654 INFO:teuthology.orchestra.run.trial004.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 27/99 2026-02-22T21:35:01.657 INFO:teuthology.orchestra.run.trial004.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 28/99 2026-02-22T21:35:01.671 INFO:teuthology.orchestra.run.trial004.stdout: Installing : python3-xmltodict-0.12.0-15.el9.noarch 29/99 2026-02-22T21:35:01.693 INFO:teuthology.orchestra.run.trial004.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 30/99 2026-02-22T21:35:01.697 INFO:teuthology.orchestra.run.trial004.stdout: Installing : python3-webob-1.8.8-2.el9.noarch 31/99 2026-02-22T21:35:01.702 INFO:teuthology.orchestra.run.trial004.stdout: Installing : python3-typing-extensions-4.15.0-1.el9.noarch 32/99 2026-02-22T21:35:01.714 INFO:teuthology.orchestra.run.trial004.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 33/99 2026-02-22T21:35:01.723 INFO:teuthology.orchestra.run.trial004.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 34/99 2026-02-22T21:35:01.729 INFO:teuthology.orchestra.run.trial004.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 35/99 2026-02-22T21:35:01.749 INFO:teuthology.orchestra.run.trial004.stdout: Installing : python3-msgpack-1.0.3-2.el9.x86_64 36/99 2026-02-22T21:35:01.757 INFO:teuthology.orchestra.run.trial004.stdout: Installing : python3-influxdb-5.3.1-1.el9.noarch 37/99 2026-02-22T21:35:01.812 INFO:teuthology.orchestra.run.trial004.stdout: Installing : python3-logutils-0.3.5-21.el9.noarch 38/99 2026-02-22T21:35:01.824 INFO:teuthology.orchestra.run.trial004.stdout: Installing : python3-pecan-1.4.2-3.el9.noarch 39/99 2026-02-22T21:35:01.840 INFO:teuthology.orchestra.run.trial004.stdout: Installing : python3-isodate-0.6.1-3.el9.noarch 40/99 2026-02-22T21:35:01.844 INFO:teuthology.orchestra.run.trial004.stdout: Installing : python3-saml-1.16.0-1.el9.noarch 41/99 2026-02-22T21:35:01.852 INFO:teuthology.orchestra.run.trial004.stdout: Installing : python3-certifi-2023.05.07-4.el9.noarch 42/99 2026-02-22T21:35:01.893 INFO:teuthology.orchestra.run.trial004.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 43/99 2026-02-22T21:35:02.223 INFO:teuthology.orchestra.run.trial004.stdout: Installing : python3-google-auth-1:2.45.0-1.el9.noarch 44/99 2026-02-22T21:35:02.234 INFO:teuthology.orchestra.run.trial004.stdout: Installing : python3-kubernetes-1:26.1.0-3.el9.noarch 45/99 2026-02-22T21:35:02.238 INFO:teuthology.orchestra.run.trial004.stdout: Installing : python3-backports-tarfile-1.2.0-1.el9.noarch 46/99 2026-02-22T21:35:02.243 INFO:teuthology.orchestra.run.trial004.stdout: Installing : python3-jaraco-context-6.0.1-3.el9.noarch 47/99 2026-02-22T21:35:02.245 INFO:teuthology.orchestra.run.trial004.stdout: Installing : python3-autocommand-2.2.2-8.el9.noarch 48/99 2026-02-22T21:35:02.273 INFO:teuthology.orchestra.run.trial004.stdout: Installing : grpc-data-1.46.7-10.el9.noarch 49/99 2026-02-22T21:35:02.317 INFO:teuthology.orchestra.run.trial004.stdout: Installing : abseil-cpp-20211102.0-4.el9.x86_64 50/99 2026-02-22T21:35:02.324 INFO:teuthology.orchestra.run.trial004.stdout: Installing : python3-grpcio-1.46.7-10.el9.x86_64 51/99 2026-02-22T21:35:02.327 INFO:teuthology.orchestra.run.trial004.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 52/99 2026-02-22T21:35:02.333 INFO:teuthology.orchestra.run.trial004.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 53/99 2026-02-22T21:35:02.336 INFO:teuthology.orchestra.run.trial004.stdout: Installing : python3-jaraco-text-4.0.0-2.el9.noarch 54/99 2026-02-22T21:35:02.343 INFO:teuthology.orchestra.run.trial004.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 55/99 2026-02-22T21:35:02.347 INFO:teuthology.orchestra.run.trial004.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 56/99 2026-02-22T21:35:02.375 INFO:teuthology.orchestra.run.trial004.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 57/99 2026-02-22T21:35:02.387 INFO:teuthology.orchestra.run.trial004.stdout: Installing : python3-protobuf-3.14.0-17.el9.noarch 58/99 2026-02-22T21:35:02.402 INFO:teuthology.orchestra.run.trial026.stdout:(1/35): ceph-radosgw-19.2.3-689.gf90f4eae.el9.x 11 MB/s | 11 MB 00:00 2026-02-22T21:35:02.422 INFO:teuthology.orchestra.run.trial004.stdout: Installing : python3-grpcio-tools-1.46.7-10.el9.x86_64 59/99 2026-02-22T21:35:02.439 INFO:teuthology.orchestra.run.trial026.stdout:(2/35): ceph-selinux-19.2.3-689.gf90f4eae.el9.x 676 kB/s | 25 kB 00:00 2026-02-22T21:35:02.495 INFO:teuthology.orchestra.run.trial026.stdout:(3/35): libcephfs2-19.2.3-689.gf90f4eae.el9.x86 17 MB/s | 999 kB 00:00 2026-02-22T21:35:02.542 INFO:teuthology.orchestra.run.trial026.stdout:(4/35): libradosstriper1-19.2.3-689.gf90f4eae.e 11 MB/s | 503 kB 00:00 2026-02-22T21:35:02.649 INFO:teuthology.orchestra.run.trial004.stdout: Installing : python3-devel-3.9.25-3.el9.x86_64 60/99 2026-02-22T21:35:02.673 INFO:teuthology.orchestra.run.trial004.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 61/99 2026-02-22T21:35:02.677 INFO:teuthology.orchestra.run.trial004.stdout: Installing : python3-jinja2-2.11.3-8.el9.noarch 62/99 2026-02-22T21:35:02.735 INFO:teuthology.orchestra.run.trial004.stdout: Installing : openblas-0.3.29-1.el9.x86_64 63/99 2026-02-22T21:35:02.737 INFO:teuthology.orchestra.run.trial004.stdout: Installing : openblas-openmp-0.3.29-1.el9.x86_64 64/99 2026-02-22T21:35:02.758 INFO:teuthology.orchestra.run.trial004.stdout: Installing : flexiblas-openblas-openmp-3.0.4-9.el9.x86_64 65/99 2026-02-22T21:35:02.825 INFO:teuthology.orchestra.run.trial026.stdout:(5/35): ceph-common-19.2.3-689.gf90f4eae.el9.x8 16 MB/s | 22 MB 00:01 2026-02-22T21:35:02.846 INFO:teuthology.orchestra.run.trial026.stdout:(6/35): python3-ceph-argparse-19.2.3-689.gf90f4 2.1 MB/s | 45 kB 00:00 2026-02-22T21:35:02.857 INFO:teuthology.orchestra.run.trial026.stdout:(7/35): librgw2-19.2.3-689.gf90f4eae.el9.x86_64 17 MB/s | 5.4 MB 00:00 2026-02-22T21:35:02.873 INFO:teuthology.orchestra.run.trial026.stdout:(8/35): python3-ceph-common-19.2.3-689.gf90f4ea 5.2 MB/s | 142 kB 00:00 2026-02-22T21:35:02.883 INFO:teuthology.orchestra.run.trial026.stdout:(9/35): python3-cephfs-19.2.3-689.gf90f4eae.el9 6.2 MB/s | 165 kB 00:00 2026-02-22T21:35:02.904 INFO:teuthology.orchestra.run.trial026.stdout:(10/35): python3-rados-19.2.3-689.gf90f4eae.el9 10 MB/s | 322 kB 00:00 2026-02-22T21:35:02.917 INFO:teuthology.orchestra.run.trial026.stdout:(11/35): python3-rbd-19.2.3-689.gf90f4eae.el9.x 9.0 MB/s | 303 kB 00:00 2026-02-22T21:35:02.933 INFO:teuthology.orchestra.run.trial026.stdout:(12/35): python3-rgw-19.2.3-689.gf90f4eae.el9.x 3.4 MB/s | 100 kB 00:00 2026-02-22T21:35:02.962 INFO:teuthology.orchestra.run.trial026.stdout:(13/35): mailcap-2.1.49-5.el9.noarch.rpm 734 kB/s | 33 kB 00:00 2026-02-22T21:35:02.988 INFO:teuthology.orchestra.run.trial026.stdout:(14/35): pciutils-3.7.0-7.el9.x86_64.rpm 3.5 MB/s | 93 kB 00:00 2026-02-22T21:35:03.016 INFO:teuthology.orchestra.run.trial026.stdout:(15/35): nvme-cli-2.16-1.el9.x86_64.rpm 14 MB/s | 1.2 MB 00:00 2026-02-22T21:35:03.029 INFO:teuthology.orchestra.run.trial026.stdout:(16/35): boost-program-options-1.75.0-13.el9.x8 2.5 MB/s | 104 kB 00:00 2026-02-22T21:35:03.047 INFO:teuthology.orchestra.run.trial026.stdout:(17/35): libpmemobj-1.12.1-1.el9.x86_64.rpm 8.7 MB/s | 160 kB 00:00 2026-02-22T21:35:03.056 INFO:teuthology.orchestra.run.trial026.stdout:(18/35): librabbitmq-0.11.0-7.el9.x86_64.rpm 4.7 MB/s | 45 kB 00:00 2026-02-22T21:35:03.059 INFO:teuthology.orchestra.run.trial026.stdout:(19/35): libnbd-1.20.3-4.el9.x86_64.rpm 3.7 MB/s | 164 kB 00:00 2026-02-22T21:35:03.079 INFO:teuthology.orchestra.run.trial026.stdout:(20/35): librdkafka-1.6.1-102.el9.x86_64.rpm 29 MB/s | 662 kB 00:00 2026-02-22T21:35:03.079 INFO:teuthology.orchestra.run.trial026.stdout:(21/35): lttng-ust-2.12.0-6.el9.x86_64.rpm 14 MB/s | 292 kB 00:00 2026-02-22T21:35:03.088 INFO:teuthology.orchestra.run.trial004.stdout: Installing : flexiblas-netlib-3.0.4-9.el9.x86_64 66/99 2026-02-22T21:35:03.089 INFO:teuthology.orchestra.run.trial026.stdout:(22/35): qatlib-service-25.08.0-2.el9.x86_64.rp 4.0 MB/s | 37 kB 00:00 2026-02-22T21:35:03.090 INFO:teuthology.orchestra.run.trial026.stdout:(23/35): qatlib-25.08.0-2.el9.x86_64.rpm 21 MB/s | 240 kB 00:00 2026-02-22T21:35:03.098 INFO:teuthology.orchestra.run.trial026.stdout:(24/35): qatzip-libs-1.3.1-1.el9.x86_64.rpm 6.8 MB/s | 66 kB 00:00 2026-02-22T21:35:03.120 INFO:teuthology.orchestra.run.trial026.stdout:(25/35): ceph-base-19.2.3-689.gf90f4eae.el9.x86 3.3 MB/s | 5.5 MB 00:01 2026-02-22T21:35:03.162 INFO:teuthology.orchestra.run.trial004.stdout: Installing : python3-numpy-1:1.23.5-2.el9.x86_64 67/99 2026-02-22T21:35:03.192 INFO:teuthology.orchestra.run.trial026.stdout:(26/35): libarrow-doc-9.0.0-15.el9.noarch.rpm 345 kB/s | 25 kB 00:00 2026-02-22T21:35:03.241 INFO:teuthology.orchestra.run.trial026.stdout:(27/35): liboath-2.6.12-1.el9.x86_64.rpm 1.0 MB/s | 49 kB 00:00 2026-02-22T21:35:03.253 INFO:teuthology.orchestra.run.trial026.stdout:(28/35): gperftools-libs-2.9.1-3.el9.x86_64.rpm 1.8 MB/s | 308 kB 00:00 2026-02-22T21:35:03.277 INFO:teuthology.orchestra.run.trial026.stdout:(29/35): libunwind-1.6.2-1.el9.x86_64.rpm 1.8 MB/s | 67 kB 00:00 2026-02-22T21:35:03.326 INFO:teuthology.orchestra.run.trial026.stdout:(30/35): parquet-libs-9.0.0-15.el9.x86_64.rpm 11 MB/s | 838 kB 00:00 2026-02-22T21:35:03.333 INFO:teuthology.orchestra.run.trial026.stdout:(31/35): re2-20211101-20.el9.x86_64.rpm 3.3 MB/s | 191 kB 00:00 2026-02-22T21:35:03.341 INFO:teuthology.orchestra.run.trial026.stdout:(32/35): libarrow-9.0.0-15.el9.x86_64.rpm 18 MB/s | 4.4 MB 00:00 2026-02-22T21:35:03.386 INFO:teuthology.orchestra.run.trial026.stdout:(33/35): thrift-0.15.0-4.el9.x86_64.rpm 26 MB/s | 1.6 MB 00:00 2026-02-22T21:35:03.502 INFO:teuthology.orchestra.run.trial026.stdout:(34/35): librbd1-19.2.3-689.gf90f4eae.el9.x86_6 20 MB/s | 3.2 MB 00:00 2026-02-22T21:35:03.750 INFO:teuthology.orchestra.run.trial026.stdout:(35/35): librados2-19.2.3-689.gf90f4eae.el9.x86 8.2 MB/s | 3.4 MB 00:00 2026-02-22T21:35:03.752 INFO:teuthology.orchestra.run.trial026.stdout:-------------------------------------------------------------------------------- 2026-02-22T21:35:03.752 INFO:teuthology.orchestra.run.trial026.stdout:Total 23 MB/s | 63 MB 00:02 2026-02-22T21:35:03.842 INFO:teuthology.orchestra.run.trial004.stdout: Installing : python3-numpy-f2py-1:1.23.5-2.el9.x86_64 68/99 2026-02-22T21:35:03.844 INFO:teuthology.orchestra.run.trial026.stdout:Running transaction check 2026-02-22T21:35:03.859 INFO:teuthology.orchestra.run.trial004.stdout: Installing : python3-scipy-1.9.3-2.el9.x86_64 69/99 2026-02-22T21:35:03.863 INFO:teuthology.orchestra.run.trial026.stdout:Transaction check succeeded. 2026-02-22T21:35:03.863 INFO:teuthology.orchestra.run.trial026.stdout:Running transaction test 2026-02-22T21:35:03.869 INFO:teuthology.orchestra.run.trial004.stdout: Installing : python3-pyparsing-2.4.7-9.el9.noarch 70/99 2026-02-22T21:35:03.875 INFO:teuthology.orchestra.run.trial004.stdout: Installing : python3-packaging-20.9-5.el9.noarch 71/99 2026-02-22T21:35:03.889 INFO:teuthology.orchestra.run.trial004.stdout: Installing : python3-ply-3.11-14.el9.noarch 72/99 2026-02-22T21:35:03.905 INFO:teuthology.orchestra.run.trial004.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 73/99 2026-02-22T21:35:03.981 INFO:teuthology.orchestra.run.trial004.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 74/99 2026-02-22T21:35:03.990 INFO:teuthology.orchestra.run.trial004.stdout: Installing : python3-cryptography-36.0.1-5.el9.x86_64 75/99 2026-02-22T21:35:04.014 INFO:teuthology.orchestra.run.trial004.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 76/99 2026-02-22T21:35:04.043 INFO:teuthology.orchestra.run.trial004.stdout: Installing : python3-cheroot-10.0.1-4.el9.noarch 77/99 2026-02-22T21:35:04.095 INFO:teuthology.orchestra.run.trial004.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 78/99 2026-02-22T21:35:04.101 INFO:teuthology.orchestra.run.trial004.stdout: Installing : python3-asyncssh-2.13.2-5.el9.noarch 79/99 2026-02-22T21:35:04.106 INFO:teuthology.orchestra.run.trial004.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 80/99 2026-02-22T21:35:04.107 INFO:teuthology.orchestra.run.trial004.stdout: Installing : libconfig-1.7.2-9.el9.x86_64 81/99 2026-02-22T21:35:04.118 INFO:teuthology.orchestra.run.trial026.stdout:Transaction test succeeded. 2026-02-22T21:35:04.119 INFO:teuthology.orchestra.run.trial026.stdout:Running transaction 2026-02-22T21:35:04.127 INFO:teuthology.orchestra.run.trial004.stdout: Running scriptlet: libstoragemgmt-1.10.1-1.el9.x86_64 82/99 2026-02-22T21:35:04.127 INFO:teuthology.orchestra.run.trial004.stdout:Creating group 'libstoragemgmt' with GID 991. 2026-02-22T21:35:04.127 INFO:teuthology.orchestra.run.trial004.stdout:Creating user 'libstoragemgmt' (daemon account for libstoragemgmt) with UID 991 and GID 991. 2026-02-22T21:35:04.127 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:35:04.137 INFO:teuthology.orchestra.run.trial004.stdout: Installing : libstoragemgmt-1.10.1-1.el9.x86_64 82/99 2026-02-22T21:35:04.164 INFO:teuthology.orchestra.run.trial004.stdout: Running scriptlet: libstoragemgmt-1.10.1-1.el9.x86_64 82/99 2026-02-22T21:35:04.164 INFO:teuthology.orchestra.run.trial004.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/libstoragemgmt.service → /usr/lib/systemd/system/libstoragemgmt.service. 2026-02-22T21:35:04.164 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:35:04.201 INFO:teuthology.orchestra.run.trial004.stdout: Installing : python3-libstoragemgmt-1.10.1-1.el9.x86_64 83/99 2026-02-22T21:35:04.265 INFO:teuthology.orchestra.run.trial004.stdout: Installing : cryptsetup-2.8.1-3.el9.x86_64 84/99 2026-02-22T21:35:04.268 INFO:teuthology.orchestra.run.trial004.stdout: Installing : ceph-volume-2:19.2.3-689.gf90f4eae.el9.noarch 85/99 2026-02-22T21:35:04.282 INFO:teuthology.orchestra.run.trial004.stdout: Running scriptlet: ceph-volume-2:19.2.3-689.gf90f4eae.el9.noarch 85/99 2026-02-22T21:35:04.282 INFO:teuthology.orchestra.run.trial004.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-22T21:35:04.282 INFO:teuthology.orchestra.run.trial004.stdout:Invalid unit name "ceph-volume@*.service" escaped as "ceph-volume@\x2a.service". 2026-02-22T21:35:04.282 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:35:04.410 INFO:teuthology.orchestra.run.trial026.stdout: Preparing : 1/1 2026-02-22T21:35:04.419 INFO:teuthology.orchestra.run.trial026.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 1/37 2026-02-22T21:35:04.422 INFO:teuthology.orchestra.run.trial026.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 2/37 2026-02-22T21:35:04.434 INFO:teuthology.orchestra.run.trial026.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 3/37 2026-02-22T21:35:04.584 INFO:teuthology.orchestra.run.trial026.stdout: Installing : thrift-0.15.0-4.el9.x86_64 4/37 2026-02-22T21:35:04.588 INFO:teuthology.orchestra.run.trial026.stdout: Upgrading : librados2-2:19.2.3-689.gf90f4eae.el9.x86_64 5/37 2026-02-22T21:35:04.616 INFO:teuthology.orchestra.run.trial026.stdout: Running scriptlet: librados2-2:19.2.3-689.gf90f4eae.el9.x86_64 5/37 2026-02-22T21:35:04.658 INFO:teuthology.orchestra.run.trial026.stdout: Installing : python3-rados-2:19.2.3-689.gf90f4eae.el9.x86_64 6/37 2026-02-22T21:35:04.658 INFO:teuthology.orchestra.run.trial026.stdout: Installing : libcephfs2-2:19.2.3-689.gf90f4eae.el9.x86_64 7/37 2026-02-22T21:35:04.669 INFO:teuthology.orchestra.run.trial026.stdout: Running scriptlet: libcephfs2-2:19.2.3-689.gf90f4eae.el9.x86_64 7/37 2026-02-22T21:35:04.673 INFO:teuthology.orchestra.run.trial026.stdout: Installing : liboath-2.6.12-1.el9.x86_64 8/37 2026-02-22T21:35:04.682 INFO:teuthology.orchestra.run.trial026.stdout: Installing : python3-ceph-argparse-2:19.2.3-689.gf90f4eae.el9.x 9/37 2026-02-22T21:35:04.704 INFO:teuthology.orchestra.run.trial026.stdout: Installing : python3-cephfs-2:19.2.3-689.gf90f4eae.el9.x86_64 10/37 2026-02-22T21:35:04.705 INFO:teuthology.orchestra.run.trial026.stdout: Installing : libradosstriper1-2:19.2.3-689.gf90f4eae.el9.x86_64 11/37 2026-02-22T21:35:04.718 INFO:teuthology.orchestra.run.trial026.stdout: Running scriptlet: libradosstriper1-2:19.2.3-689.gf90f4eae.el9.x86_64 11/37 2026-02-22T21:35:04.756 INFO:teuthology.orchestra.run.trial026.stdout: Installing : re2-1:20211101-20.el9.x86_64 12/37 2026-02-22T21:35:04.760 INFO:teuthology.orchestra.run.trial026.stdout: Installing : libarrow-9.0.0-15.el9.x86_64 13/37 2026-02-22T21:35:04.766 INFO:teuthology.orchestra.run.trial026.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 14/37 2026-02-22T21:35:04.768 INFO:teuthology.orchestra.run.trial026.stdout: Installing : gperftools-libs-2.9.1-3.el9.x86_64 15/37 2026-02-22T21:35:04.771 INFO:teuthology.orchestra.run.trial026.stdout: Installing : libarrow-doc-9.0.0-15.el9.noarch 16/37 2026-02-22T21:35:04.777 INFO:teuthology.orchestra.run.trial026.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 17/37 2026-02-22T21:35:04.916 INFO:teuthology.orchestra.run.trial026.stdout: Installing : libnbd-1.20.3-4.el9.x86_64 18/37 2026-02-22T21:35:04.917 INFO:teuthology.orchestra.run.trial026.stdout: Upgrading : librbd1-2:19.2.3-689.gf90f4eae.el9.x86_64 19/37 2026-02-22T21:35:04.945 INFO:teuthology.orchestra.run.trial026.stdout: Running scriptlet: librbd1-2:19.2.3-689.gf90f4eae.el9.x86_64 19/37 2026-02-22T21:35:04.949 INFO:teuthology.orchestra.run.trial026.stdout: Installing : python3-rbd-2:19.2.3-689.gf90f4eae.el9.x86_64 20/37 2026-02-22T21:35:04.955 INFO:teuthology.orchestra.run.trial026.stdout: Installing : boost-program-options-1.75.0-13.el9.x86_64 21/37 2026-02-22T21:35:04.979 INFO:teuthology.orchestra.run.trial004.stdout: Installing : ceph-osd-2:19.2.3-689.gf90f4eae.el9.x86_64 86/99 2026-02-22T21:35:05.003 INFO:teuthology.orchestra.run.trial004.stdout: Running scriptlet: ceph-osd-2:19.2.3-689.gf90f4eae.el9.x86_64 86/99 2026-02-22T21:35:05.003 INFO:teuthology.orchestra.run.trial004.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-22T21:35:05.003 INFO:teuthology.orchestra.run.trial004.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2026-02-22T21:35:05.003 INFO:teuthology.orchestra.run.trial004.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2026-02-22T21:35:05.003 INFO:teuthology.orchestra.run.trial004.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2026-02-22T21:35:05.003 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:35:05.060 INFO:teuthology.orchestra.run.trial004.stdout: Running scriptlet: cephadm-2:19.2.3-689.gf90f4eae.el9.noarch 87/99 2026-02-22T21:35:05.062 INFO:teuthology.orchestra.run.trial004.stdout: Installing : cephadm-2:19.2.3-689.gf90f4eae.el9.noarch 87/99 2026-02-22T21:35:05.066 INFO:teuthology.orchestra.run.trial004.stdout: Installing : ceph-prometheus-alerts-2:19.2.3-689.gf90f4eae.el9. 88/99 2026-02-22T21:35:05.086 INFO:teuthology.orchestra.run.trial004.stdout: Installing : ceph-grafana-dashboards-2:19.2.3-689.gf90f4eae.el9 89/99 2026-02-22T21:35:05.087 INFO:teuthology.orchestra.run.trial004.stdout: Installing : ceph-mgr-cephadm-2:19.2.3-689.gf90f4eae.el9.noarch 90/99 2026-02-22T21:35:05.179 INFO:teuthology.orchestra.run.trial026.stdout: Installing : parquet-libs-9.0.0-15.el9.x86_64 22/37 2026-02-22T21:35:05.180 INFO:teuthology.orchestra.run.trial026.stdout: Installing : librgw2-2:19.2.3-689.gf90f4eae.el9.x86_64 23/37 2026-02-22T21:35:05.196 INFO:teuthology.orchestra.run.trial026.stdout: Running scriptlet: librgw2-2:19.2.3-689.gf90f4eae.el9.x86_64 23/37 2026-02-22T21:35:05.200 INFO:teuthology.orchestra.run.trial026.stdout: Installing : python3-rgw-2:19.2.3-689.gf90f4eae.el9.x86_64 24/37 2026-02-22T21:35:05.205 INFO:teuthology.orchestra.run.trial026.stdout: Installing : pciutils-3.7.0-7.el9.x86_64 25/37 2026-02-22T21:35:05.208 INFO:teuthology.orchestra.run.trial026.stdout: Installing : qatlib-25.08.0-2.el9.x86_64 26/37 2026-02-22T21:35:05.209 INFO:teuthology.orchestra.run.trial026.stdout: Installing : qatlib-service-25.08.0-2.el9.x86_64 27/37 2026-02-22T21:35:05.226 INFO:teuthology.orchestra.run.trial026.stdout: Running scriptlet: qatlib-service-25.08.0-2.el9.x86_64 27/37 2026-02-22T21:35:05.342 INFO:teuthology.orchestra.run.trial026.stdout: Installing : qatzip-libs-1.3.1-1.el9.x86_64 28/37 2026-02-22T21:35:05.344 INFO:teuthology.orchestra.run.trial026.stdout: Installing : nvme-cli-2.16-1.el9.x86_64 29/37 2026-02-22T21:35:05.584 INFO:teuthology.orchestra.run.trial004.stdout: Running scriptlet: ceph-mgr-cephadm-2:19.2.3-689.gf90f4eae.el9.noarch 90/99 2026-02-22T21:35:05.587 INFO:teuthology.orchestra.run.trial004.stdout: Installing : ceph-mgr-dashboard-2:19.2.3-689.gf90f4eae.el9.noar 91/99 2026-02-22T21:35:05.619 INFO:teuthology.orchestra.run.trial026.stdout: Running scriptlet: nvme-cli-2.16-1.el9.x86_64 29/37 2026-02-22T21:35:05.619 INFO:teuthology.orchestra.run.trial026.stdout:Created symlink /etc/systemd/system/default.target.wants/nvmefc-boot-connections.service → /usr/lib/systemd/system/nvmefc-boot-connections.service. 2026-02-22T21:35:05.619 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:35:05.654 INFO:teuthology.orchestra.run.trial026.stdout: Installing : mailcap-2.1.49-5.el9.noarch 30/37 2026-02-22T21:35:05.656 INFO:teuthology.orchestra.run.trial026.stdout: Installing : python3-ceph-common-2:19.2.3-689.gf90f4eae.el9.x86 31/37 2026-02-22T21:35:06.041 INFO:teuthology.orchestra.run.trial004.stdout: Running scriptlet: ceph-mgr-dashboard-2:19.2.3-689.gf90f4eae.el9.noar 91/99 2026-02-22T21:35:06.042 INFO:teuthology.orchestra.run.trial004.stdout: Installing : ceph-mgr-diskprediction-local-2:19.2.3-689.gf90f4e 92/99 2026-02-22T21:35:06.054 INFO:teuthology.orchestra.run.trial004.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:19.2.3-689.gf90f4e 92/99 2026-02-22T21:35:06.054 INFO:teuthology.orchestra.run.trial004.stdout: Installing : ceph-mgr-k8sevents-2:19.2.3-689.gf90f4eae.el9.noar 93/99 2026-02-22T21:35:06.111 INFO:teuthology.orchestra.run.trial004.stdout: Running scriptlet: ceph-mgr-k8sevents-2:19.2.3-689.gf90f4eae.el9.noar 93/99 2026-02-22T21:35:06.161 INFO:teuthology.orchestra.run.trial004.stdout: Installing : ceph-mgr-modules-core-2:19.2.3-689.gf90f4eae.el9.n 94/99 2026-02-22T21:35:06.162 INFO:teuthology.orchestra.run.trial004.stdout: Installing : ceph-mgr-2:19.2.3-689.gf90f4eae.el9.x86_64 95/99 2026-02-22T21:35:06.183 INFO:teuthology.orchestra.run.trial004.stdout: Running scriptlet: ceph-mgr-2:19.2.3-689.gf90f4eae.el9.x86_64 95/99 2026-02-22T21:35:06.183 INFO:teuthology.orchestra.run.trial004.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-22T21:35:06.183 INFO:teuthology.orchestra.run.trial004.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2026-02-22T21:35:06.184 INFO:teuthology.orchestra.run.trial004.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2026-02-22T21:35:06.184 INFO:teuthology.orchestra.run.trial004.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2026-02-22T21:35:06.184 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:35:06.195 INFO:teuthology.orchestra.run.trial004.stdout: Installing : ceph-mgr-rook-2:19.2.3-689.gf90f4eae.el9.noarch 96/99 2026-02-22T21:35:06.411 INFO:teuthology.orchestra.run.trial004.stdout: Running scriptlet: ceph-mgr-rook-2:19.2.3-689.gf90f4eae.el9.noarch 96/99 2026-02-22T21:35:06.412 INFO:teuthology.orchestra.run.trial004.stdout: Installing : ceph-mon-2:19.2.3-689.gf90f4eae.el9.x86_64 97/99 2026-02-22T21:35:06.433 INFO:teuthology.orchestra.run.trial004.stdout: Running scriptlet: ceph-mon-2:19.2.3-689.gf90f4eae.el9.x86_64 97/99 2026-02-22T21:35:06.433 INFO:teuthology.orchestra.run.trial004.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-22T21:35:06.434 INFO:teuthology.orchestra.run.trial004.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2026-02-22T21:35:06.434 INFO:teuthology.orchestra.run.trial004.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2026-02-22T21:35:06.434 INFO:teuthology.orchestra.run.trial004.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2026-02-22T21:35:06.434 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:35:06.535 INFO:teuthology.orchestra.run.trial004.stdout: Installing : ceph-mds-2:19.2.3-689.gf90f4eae.el9.x86_64 98/99 2026-02-22T21:35:06.556 INFO:teuthology.orchestra.run.trial004.stdout: Running scriptlet: ceph-mds-2:19.2.3-689.gf90f4eae.el9.x86_64 98/99 2026-02-22T21:35:06.556 INFO:teuthology.orchestra.run.trial004.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-22T21:35:06.557 INFO:teuthology.orchestra.run.trial004.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2026-02-22T21:35:06.557 INFO:teuthology.orchestra.run.trial004.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2026-02-22T21:35:06.557 INFO:teuthology.orchestra.run.trial004.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2026-02-22T21:35:06.557 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:35:06.575 INFO:teuthology.orchestra.run.trial004.stdout: Installing : ceph-2:19.2.3-689.gf90f4eae.el9.x86_64 99/99 2026-02-22T21:35:06.637 INFO:teuthology.orchestra.run.trial026.stdout: Running scriptlet: ceph-common-2:19.2.3-689.gf90f4eae.el9.x86_64 32/37 2026-02-22T21:35:06.639 INFO:teuthology.orchestra.run.trial026.stdout: Installing : ceph-common-2:19.2.3-689.gf90f4eae.el9.x86_64 32/37 2026-02-22T21:35:06.927 INFO:teuthology.orchestra.run.trial026.stdout: Running scriptlet: ceph-common-2:19.2.3-689.gf90f4eae.el9.x86_64 32/37 2026-02-22T21:35:06.930 INFO:teuthology.orchestra.run.trial026.stdout: Installing : ceph-base-2:19.2.3-689.gf90f4eae.el9.x86_64 33/37 2026-02-22T21:35:06.968 INFO:teuthology.orchestra.run.trial026.stdout: Running scriptlet: ceph-base-2:19.2.3-689.gf90f4eae.el9.x86_64 33/37 2026-02-22T21:35:06.969 INFO:teuthology.orchestra.run.trial026.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2026-02-22T21:35:06.969 INFO:teuthology.orchestra.run.trial026.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2026-02-22T21:35:06.969 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:35:06.972 INFO:teuthology.orchestra.run.trial026.stdout: Installing : ceph-selinux-2:19.2.3-689.gf90f4eae.el9.x86_64 34/37 2026-02-22T21:35:07.225 INFO:teuthology.orchestra.run.trial004.stdout: Running scriptlet: ceph-2:19.2.3-689.gf90f4eae.el9.x86_64 99/99 2026-02-22T21:35:07.225 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : ceph-2:19.2.3-689.gf90f4eae.el9.x86_64 1/99 2026-02-22T21:35:07.225 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : ceph-mds-2:19.2.3-689.gf90f4eae.el9.x86_64 2/99 2026-02-22T21:35:07.225 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : ceph-mgr-2:19.2.3-689.gf90f4eae.el9.x86_64 3/99 2026-02-22T21:35:07.225 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : ceph-mon-2:19.2.3-689.gf90f4eae.el9.x86_64 4/99 2026-02-22T21:35:07.225 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : ceph-osd-2:19.2.3-689.gf90f4eae.el9.x86_64 5/99 2026-02-22T21:35:07.225 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : ceph-grafana-dashboards-2:19.2.3-689.gf90f4eae.el9 6/99 2026-02-22T21:35:07.225 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : ceph-mgr-cephadm-2:19.2.3-689.gf90f4eae.el9.noarch 7/99 2026-02-22T21:35:07.225 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : ceph-mgr-dashboard-2:19.2.3-689.gf90f4eae.el9.noar 8/99 2026-02-22T21:35:07.225 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : ceph-mgr-diskprediction-local-2:19.2.3-689.gf90f4e 9/99 2026-02-22T21:35:07.225 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : ceph-mgr-k8sevents-2:19.2.3-689.gf90f4eae.el9.noar 10/99 2026-02-22T21:35:07.225 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : ceph-mgr-modules-core-2:19.2.3-689.gf90f4eae.el9.n 11/99 2026-02-22T21:35:07.225 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : ceph-mgr-rook-2:19.2.3-689.gf90f4eae.el9.noarch 12/99 2026-02-22T21:35:07.226 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : ceph-prometheus-alerts-2:19.2.3-689.gf90f4eae.el9. 13/99 2026-02-22T21:35:07.226 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : ceph-volume-2:19.2.3-689.gf90f4eae.el9.noarch 14/99 2026-02-22T21:35:07.226 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : cephadm-2:19.2.3-689.gf90f4eae.el9.noarch 15/99 2026-02-22T21:35:07.226 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : cryptsetup-2.8.1-3.el9.x86_64 16/99 2026-02-22T21:35:07.226 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : ledmon-libs-1.1.0-3.el9.x86_64 17/99 2026-02-22T21:35:07.226 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : libconfig-1.7.2-9.el9.x86_64 18/99 2026-02-22T21:35:07.226 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : libgfortran-11.5.0-14.el9.x86_64 19/99 2026-02-22T21:35:07.226 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : libquadmath-11.5.0-14.el9.x86_64 20/99 2026-02-22T21:35:07.226 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 21/99 2026-02-22T21:35:07.226 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : python3-cryptography-36.0.1-5.el9.x86_64 22/99 2026-02-22T21:35:07.226 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : python3-ply-3.11-14.el9.noarch 23/99 2026-02-22T21:35:07.226 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 24/99 2026-02-22T21:35:07.226 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : python3-pyparsing-2.4.7-9.el9.noarch 25/99 2026-02-22T21:35:07.226 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : python3-requests-2.25.1-10.el9.noarch 26/99 2026-02-22T21:35:07.226 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : python3-urllib3-1.26.5-7.el9.noarch 27/99 2026-02-22T21:35:07.226 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : unzip-6.0-59.el9.x86_64 28/99 2026-02-22T21:35:07.226 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : zip-3.0-35.el9.x86_64 29/99 2026-02-22T21:35:07.226 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : flexiblas-3.0.4-9.el9.x86_64 30/99 2026-02-22T21:35:07.226 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : flexiblas-netlib-3.0.4-9.el9.x86_64 31/99 2026-02-22T21:35:07.226 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-9.el9.x86_64 32/99 2026-02-22T21:35:07.226 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : libstoragemgmt-1.10.1-1.el9.x86_64 33/99 2026-02-22T21:35:07.226 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : lua-5.4.4-4.el9.x86_64 34/99 2026-02-22T21:35:07.226 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : openblas-0.3.29-1.el9.x86_64 35/99 2026-02-22T21:35:07.226 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : openblas-openmp-0.3.29-1.el9.x86_64 36/99 2026-02-22T21:35:07.226 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : protobuf-3.14.0-17.el9.x86_64 37/99 2026-02-22T21:35:07.226 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 38/99 2026-02-22T21:35:07.226 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : python3-devel-3.9.25-3.el9.x86_64 39/99 2026-02-22T21:35:07.226 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : python3-jinja2-2.11.3-8.el9.noarch 40/99 2026-02-22T21:35:07.226 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : python3-libstoragemgmt-1.10.1-1.el9.x86_64 41/99 2026-02-22T21:35:07.226 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : python3-lxml-4.6.5-3.el9.x86_64 42/99 2026-02-22T21:35:07.226 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : python3-mako-1.1.4-6.el9.noarch 43/99 2026-02-22T21:35:07.226 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 44/99 2026-02-22T21:35:07.226 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : python3-numpy-1:1.23.5-2.el9.x86_64 45/99 2026-02-22T21:35:07.226 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : python3-numpy-f2py-1:1.23.5-2.el9.x86_64 46/99 2026-02-22T21:35:07.226 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : python3-packaging-20.9-5.el9.noarch 47/99 2026-02-22T21:35:07.226 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : python3-protobuf-3.14.0-17.el9.noarch 48/99 2026-02-22T21:35:07.226 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : python3-pyasn1-0.4.8-6.el9.noarch 49/99 2026-02-22T21:35:07.227 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : python3-pyasn1-modules-0.4.8-6.el9.noarch 50/99 2026-02-22T21:35:07.227 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 51/99 2026-02-22T21:35:07.227 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : python3-scipy-1.9.3-2.el9.x86_64 52/99 2026-02-22T21:35:07.227 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 53/99 2026-02-22T21:35:07.227 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : xmlsec1-1.2.29-13.el9.x86_64 54/99 2026-02-22T21:35:07.227 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : xmlsec1-openssl-1.2.29-13.el9.x86_64 55/99 2026-02-22T21:35:07.227 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : lua-devel-5.4.4-4.el9.x86_64 56/99 2026-02-22T21:35:07.227 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : protobuf-compiler-3.14.0-17.el9.x86_64 57/99 2026-02-22T21:35:07.227 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : abseil-cpp-20211102.0-4.el9.x86_64 58/99 2026-02-22T21:35:07.227 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : grpc-data-1.46.7-10.el9.noarch 59/99 2026-02-22T21:35:07.227 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : luarocks-3.9.2-5.el9.noarch 60/99 2026-02-22T21:35:07.227 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : python3-asyncssh-2.13.2-5.el9.noarch 61/99 2026-02-22T21:35:07.227 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : python3-autocommand-2.2.2-8.el9.noarch 62/99 2026-02-22T21:35:07.227 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : python3-backports-tarfile-1.2.0-1.el9.noarch 63/99 2026-02-22T21:35:07.227 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 64/99 2026-02-22T21:35:07.227 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 65/99 2026-02-22T21:35:07.227 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : python3-certifi-2023.05.07-4.el9.noarch 66/99 2026-02-22T21:35:07.227 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : python3-cheroot-10.0.1-4.el9.noarch 67/99 2026-02-22T21:35:07.227 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 68/99 2026-02-22T21:35:07.227 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : python3-google-auth-1:2.45.0-1.el9.noarch 69/99 2026-02-22T21:35:07.227 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : python3-grpcio-1.46.7-10.el9.x86_64 70/99 2026-02-22T21:35:07.227 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : python3-grpcio-tools-1.46.7-10.el9.x86_64 71/99 2026-02-22T21:35:07.227 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : python3-influxdb-5.3.1-1.el9.noarch 72/99 2026-02-22T21:35:07.227 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : python3-isodate-0.6.1-3.el9.noarch 73/99 2026-02-22T21:35:07.227 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 74/99 2026-02-22T21:35:07.227 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 75/99 2026-02-22T21:35:07.227 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 76/99 2026-02-22T21:35:07.227 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : python3-jaraco-context-6.0.1-3.el9.noarch 77/99 2026-02-22T21:35:07.227 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 78/99 2026-02-22T21:35:07.227 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : python3-jaraco-text-4.0.0-2.el9.noarch 79/99 2026-02-22T21:35:07.227 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : python3-kubernetes-1:26.1.0-3.el9.noarch 80/99 2026-02-22T21:35:07.227 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : python3-logutils-0.3.5-21.el9.noarch 81/99 2026-02-22T21:35:07.227 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 82/99 2026-02-22T21:35:07.227 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : python3-msgpack-1.0.3-2.el9.x86_64 83/99 2026-02-22T21:35:07.227 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 84/99 2026-02-22T21:35:07.227 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : python3-pecan-1.4.2-3.el9.noarch 85/99 2026-02-22T21:35:07.227 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 86/99 2026-02-22T21:35:07.227 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 87/99 2026-02-22T21:35:07.227 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 88/99 2026-02-22T21:35:07.227 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 89/99 2026-02-22T21:35:07.227 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 90/99 2026-02-22T21:35:07.227 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : python3-saml-1.16.0-1.el9.noarch 91/99 2026-02-22T21:35:07.227 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 92/99 2026-02-22T21:35:07.228 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : python3-typing-extensions-4.15.0-1.el9.noarch 93/99 2026-02-22T21:35:07.228 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : python3-webob-1.8.8-2.el9.noarch 94/99 2026-02-22T21:35:07.228 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 95/99 2026-02-22T21:35:07.228 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : python3-werkzeug-2.0.3-3.el9.1.noarch 96/99 2026-02-22T21:35:07.228 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : python3-xmlsec-1.3.13-1.el9.x86_64 97/99 2026-02-22T21:35:07.228 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : python3-xmltodict-0.12.0-15.el9.noarch 98/99 2026-02-22T21:35:07.293 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 99/99 2026-02-22T21:35:07.293 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:35:07.293 INFO:teuthology.orchestra.run.trial004.stdout:Installed: 2026-02-22T21:35:07.293 INFO:teuthology.orchestra.run.trial004.stdout: abseil-cpp-20211102.0-4.el9.x86_64 2026-02-22T21:35:07.293 INFO:teuthology.orchestra.run.trial004.stdout: ceph-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:35:07.293 INFO:teuthology.orchestra.run.trial004.stdout: ceph-grafana-dashboards-2:19.2.3-689.gf90f4eae.el9.noarch 2026-02-22T21:35:07.293 INFO:teuthology.orchestra.run.trial004.stdout: ceph-mds-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:35:07.293 INFO:teuthology.orchestra.run.trial004.stdout: ceph-mgr-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:35:07.294 INFO:teuthology.orchestra.run.trial004.stdout: ceph-mgr-cephadm-2:19.2.3-689.gf90f4eae.el9.noarch 2026-02-22T21:35:07.294 INFO:teuthology.orchestra.run.trial004.stdout: ceph-mgr-dashboard-2:19.2.3-689.gf90f4eae.el9.noarch 2026-02-22T21:35:07.294 INFO:teuthology.orchestra.run.trial004.stdout: ceph-mgr-diskprediction-local-2:19.2.3-689.gf90f4eae.el9.noarch 2026-02-22T21:35:07.294 INFO:teuthology.orchestra.run.trial004.stdout: ceph-mgr-k8sevents-2:19.2.3-689.gf90f4eae.el9.noarch 2026-02-22T21:35:07.294 INFO:teuthology.orchestra.run.trial004.stdout: ceph-mgr-modules-core-2:19.2.3-689.gf90f4eae.el9.noarch 2026-02-22T21:35:07.294 INFO:teuthology.orchestra.run.trial004.stdout: ceph-mgr-rook-2:19.2.3-689.gf90f4eae.el9.noarch 2026-02-22T21:35:07.294 INFO:teuthology.orchestra.run.trial004.stdout: ceph-mon-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:35:07.294 INFO:teuthology.orchestra.run.trial004.stdout: ceph-osd-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:35:07.294 INFO:teuthology.orchestra.run.trial004.stdout: ceph-prometheus-alerts-2:19.2.3-689.gf90f4eae.el9.noarch 2026-02-22T21:35:07.294 INFO:teuthology.orchestra.run.trial004.stdout: ceph-volume-2:19.2.3-689.gf90f4eae.el9.noarch 2026-02-22T21:35:07.294 INFO:teuthology.orchestra.run.trial004.stdout: cephadm-2:19.2.3-689.gf90f4eae.el9.noarch 2026-02-22T21:35:07.294 INFO:teuthology.orchestra.run.trial004.stdout: cryptsetup-2.8.1-3.el9.x86_64 2026-02-22T21:35:07.294 INFO:teuthology.orchestra.run.trial004.stdout: flexiblas-3.0.4-9.el9.x86_64 2026-02-22T21:35:07.294 INFO:teuthology.orchestra.run.trial004.stdout: flexiblas-netlib-3.0.4-9.el9.x86_64 2026-02-22T21:35:07.294 INFO:teuthology.orchestra.run.trial004.stdout: flexiblas-openblas-openmp-3.0.4-9.el9.x86_64 2026-02-22T21:35:07.294 INFO:teuthology.orchestra.run.trial004.stdout: grpc-data-1.46.7-10.el9.noarch 2026-02-22T21:35:07.294 INFO:teuthology.orchestra.run.trial004.stdout: ledmon-libs-1.1.0-3.el9.x86_64 2026-02-22T21:35:07.294 INFO:teuthology.orchestra.run.trial004.stdout: libconfig-1.7.2-9.el9.x86_64 2026-02-22T21:35:07.294 INFO:teuthology.orchestra.run.trial004.stdout: libgfortran-11.5.0-14.el9.x86_64 2026-02-22T21:35:07.294 INFO:teuthology.orchestra.run.trial004.stdout: libquadmath-11.5.0-14.el9.x86_64 2026-02-22T21:35:07.294 INFO:teuthology.orchestra.run.trial004.stdout: libstoragemgmt-1.10.1-1.el9.x86_64 2026-02-22T21:35:07.294 INFO:teuthology.orchestra.run.trial004.stdout: lua-5.4.4-4.el9.x86_64 2026-02-22T21:35:07.294 INFO:teuthology.orchestra.run.trial004.stdout: lua-devel-5.4.4-4.el9.x86_64 2026-02-22T21:35:07.294 INFO:teuthology.orchestra.run.trial004.stdout: luarocks-3.9.2-5.el9.noarch 2026-02-22T21:35:07.294 INFO:teuthology.orchestra.run.trial004.stdout: openblas-0.3.29-1.el9.x86_64 2026-02-22T21:35:07.294 INFO:teuthology.orchestra.run.trial004.stdout: openblas-openmp-0.3.29-1.el9.x86_64 2026-02-22T21:35:07.294 INFO:teuthology.orchestra.run.trial004.stdout: protobuf-3.14.0-17.el9.x86_64 2026-02-22T21:35:07.294 INFO:teuthology.orchestra.run.trial004.stdout: protobuf-compiler-3.14.0-17.el9.x86_64 2026-02-22T21:35:07.294 INFO:teuthology.orchestra.run.trial004.stdout: python3-asyncssh-2.13.2-5.el9.noarch 2026-02-22T21:35:07.294 INFO:teuthology.orchestra.run.trial004.stdout: python3-autocommand-2.2.2-8.el9.noarch 2026-02-22T21:35:07.295 INFO:teuthology.orchestra.run.trial004.stdout: python3-babel-2.9.1-2.el9.noarch 2026-02-22T21:35:07.295 INFO:teuthology.orchestra.run.trial004.stdout: python3-backports-tarfile-1.2.0-1.el9.noarch 2026-02-22T21:35:07.295 INFO:teuthology.orchestra.run.trial004.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2026-02-22T21:35:07.295 INFO:teuthology.orchestra.run.trial004.stdout: python3-cachetools-4.2.4-1.el9.noarch 2026-02-22T21:35:07.295 INFO:teuthology.orchestra.run.trial004.stdout: python3-certifi-2023.05.07-4.el9.noarch 2026-02-22T21:35:07.295 INFO:teuthology.orchestra.run.trial004.stdout: python3-cffi-1.14.5-5.el9.x86_64 2026-02-22T21:35:07.295 INFO:teuthology.orchestra.run.trial004.stdout: python3-cheroot-10.0.1-4.el9.noarch 2026-02-22T21:35:07.295 INFO:teuthology.orchestra.run.trial004.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2026-02-22T21:35:07.295 INFO:teuthology.orchestra.run.trial004.stdout: python3-cryptography-36.0.1-5.el9.x86_64 2026-02-22T21:35:07.295 INFO:teuthology.orchestra.run.trial004.stdout: python3-devel-3.9.25-3.el9.x86_64 2026-02-22T21:35:07.295 INFO:teuthology.orchestra.run.trial004.stdout: python3-google-auth-1:2.45.0-1.el9.noarch 2026-02-22T21:35:07.295 INFO:teuthology.orchestra.run.trial004.stdout: python3-grpcio-1.46.7-10.el9.x86_64 2026-02-22T21:35:07.295 INFO:teuthology.orchestra.run.trial004.stdout: python3-grpcio-tools-1.46.7-10.el9.x86_64 2026-02-22T21:35:07.295 INFO:teuthology.orchestra.run.trial004.stdout: python3-influxdb-5.3.1-1.el9.noarch 2026-02-22T21:35:07.295 INFO:teuthology.orchestra.run.trial004.stdout: python3-isodate-0.6.1-3.el9.noarch 2026-02-22T21:35:07.295 INFO:teuthology.orchestra.run.trial004.stdout: python3-jaraco-8.2.1-3.el9.noarch 2026-02-22T21:35:07.295 INFO:teuthology.orchestra.run.trial004.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2026-02-22T21:35:07.295 INFO:teuthology.orchestra.run.trial004.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2026-02-22T21:35:07.295 INFO:teuthology.orchestra.run.trial004.stdout: python3-jaraco-context-6.0.1-3.el9.noarch 2026-02-22T21:35:07.295 INFO:teuthology.orchestra.run.trial004.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2026-02-22T21:35:07.295 INFO:teuthology.orchestra.run.trial004.stdout: python3-jaraco-text-4.0.0-2.el9.noarch 2026-02-22T21:35:07.295 INFO:teuthology.orchestra.run.trial004.stdout: python3-jinja2-2.11.3-8.el9.noarch 2026-02-22T21:35:07.295 INFO:teuthology.orchestra.run.trial004.stdout: python3-kubernetes-1:26.1.0-3.el9.noarch 2026-02-22T21:35:07.295 INFO:teuthology.orchestra.run.trial004.stdout: python3-libstoragemgmt-1.10.1-1.el9.x86_64 2026-02-22T21:35:07.295 INFO:teuthology.orchestra.run.trial004.stdout: python3-logutils-0.3.5-21.el9.noarch 2026-02-22T21:35:07.295 INFO:teuthology.orchestra.run.trial004.stdout: python3-lxml-4.6.5-3.el9.x86_64 2026-02-22T21:35:07.295 INFO:teuthology.orchestra.run.trial004.stdout: python3-mako-1.1.4-6.el9.noarch 2026-02-22T21:35:07.295 INFO:teuthology.orchestra.run.trial004.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2026-02-22T21:35:07.295 INFO:teuthology.orchestra.run.trial004.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2026-02-22T21:35:07.295 INFO:teuthology.orchestra.run.trial004.stdout: python3-msgpack-1.0.3-2.el9.x86_64 2026-02-22T21:35:07.295 INFO:teuthology.orchestra.run.trial004.stdout: python3-natsort-7.1.1-5.el9.noarch 2026-02-22T21:35:07.296 INFO:teuthology.orchestra.run.trial004.stdout: python3-numpy-1:1.23.5-2.el9.x86_64 2026-02-22T21:35:07.296 INFO:teuthology.orchestra.run.trial004.stdout: python3-numpy-f2py-1:1.23.5-2.el9.x86_64 2026-02-22T21:35:07.296 INFO:teuthology.orchestra.run.trial004.stdout: python3-packaging-20.9-5.el9.noarch 2026-02-22T21:35:07.296 INFO:teuthology.orchestra.run.trial004.stdout: python3-pecan-1.4.2-3.el9.noarch 2026-02-22T21:35:07.296 INFO:teuthology.orchestra.run.trial004.stdout: python3-ply-3.11-14.el9.noarch 2026-02-22T21:35:07.296 INFO:teuthology.orchestra.run.trial004.stdout: python3-portend-3.1.0-2.el9.noarch 2026-02-22T21:35:07.296 INFO:teuthology.orchestra.run.trial004.stdout: python3-protobuf-3.14.0-17.el9.noarch 2026-02-22T21:35:07.296 INFO:teuthology.orchestra.run.trial004.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2026-02-22T21:35:07.296 INFO:teuthology.orchestra.run.trial004.stdout: python3-pyasn1-0.4.8-6.el9.noarch 2026-02-22T21:35:07.296 INFO:teuthology.orchestra.run.trial004.stdout: python3-pyasn1-modules-0.4.8-6.el9.noarch 2026-02-22T21:35:07.296 INFO:teuthology.orchestra.run.trial004.stdout: python3-pycparser-2.20-6.el9.noarch 2026-02-22T21:35:07.296 INFO:teuthology.orchestra.run.trial004.stdout: python3-pyparsing-2.4.7-9.el9.noarch 2026-02-22T21:35:07.296 INFO:teuthology.orchestra.run.trial004.stdout: python3-repoze-lru-0.7-16.el9.noarch 2026-02-22T21:35:07.296 INFO:teuthology.orchestra.run.trial004.stdout: python3-requests-2.25.1-10.el9.noarch 2026-02-22T21:35:07.296 INFO:teuthology.orchestra.run.trial004.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2026-02-22T21:35:07.296 INFO:teuthology.orchestra.run.trial004.stdout: python3-routes-2.5.1-5.el9.noarch 2026-02-22T21:35:07.296 INFO:teuthology.orchestra.run.trial004.stdout: python3-rsa-4.9-2.el9.noarch 2026-02-22T21:35:07.296 INFO:teuthology.orchestra.run.trial004.stdout: python3-saml-1.16.0-1.el9.noarch 2026-02-22T21:35:07.296 INFO:teuthology.orchestra.run.trial004.stdout: python3-scipy-1.9.3-2.el9.x86_64 2026-02-22T21:35:07.296 INFO:teuthology.orchestra.run.trial004.stdout: python3-tempora-5.0.0-2.el9.noarch 2026-02-22T21:35:07.296 INFO:teuthology.orchestra.run.trial004.stdout: python3-toml-0.10.2-6.el9.noarch 2026-02-22T21:35:07.296 INFO:teuthology.orchestra.run.trial004.stdout: python3-typing-extensions-4.15.0-1.el9.noarch 2026-02-22T21:35:07.296 INFO:teuthology.orchestra.run.trial004.stdout: python3-urllib3-1.26.5-7.el9.noarch 2026-02-22T21:35:07.296 INFO:teuthology.orchestra.run.trial004.stdout: python3-webob-1.8.8-2.el9.noarch 2026-02-22T21:35:07.296 INFO:teuthology.orchestra.run.trial004.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2026-02-22T21:35:07.296 INFO:teuthology.orchestra.run.trial004.stdout: python3-werkzeug-2.0.3-3.el9.1.noarch 2026-02-22T21:35:07.296 INFO:teuthology.orchestra.run.trial004.stdout: python3-xmlsec-1.3.13-1.el9.x86_64 2026-02-22T21:35:07.296 INFO:teuthology.orchestra.run.trial004.stdout: python3-xmltodict-0.12.0-15.el9.noarch 2026-02-22T21:35:07.296 INFO:teuthology.orchestra.run.trial004.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2026-02-22T21:35:07.296 INFO:teuthology.orchestra.run.trial004.stdout: unzip-6.0-59.el9.x86_64 2026-02-22T21:35:07.296 INFO:teuthology.orchestra.run.trial004.stdout: xmlsec1-1.2.29-13.el9.x86_64 2026-02-22T21:35:07.296 INFO:teuthology.orchestra.run.trial004.stdout: xmlsec1-openssl-1.2.29-13.el9.x86_64 2026-02-22T21:35:07.297 INFO:teuthology.orchestra.run.trial004.stdout: zip-3.0-35.el9.x86_64 2026-02-22T21:35:07.297 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:35:07.297 INFO:teuthology.orchestra.run.trial004.stdout:Complete! 2026-02-22T21:35:07.347 DEBUG:teuthology.orchestra.run.trial004:> sudo yum -y install ceph-base 2026-02-22T21:35:07.659 INFO:teuthology.orchestra.run.trial004.stdout:Last metadata expiration check: 0:00:32 ago on Sun 22 Feb 2026 09:34:35 PM UTC. 2026-02-22T21:35:07.723 INFO:teuthology.orchestra.run.trial004.stdout:Package ceph-base-2:19.2.3-689.gf90f4eae.el9.x86_64 is already installed. 2026-02-22T21:35:07.742 INFO:teuthology.orchestra.run.trial004.stdout:Dependencies resolved. 2026-02-22T21:35:07.742 INFO:teuthology.orchestra.run.trial004.stdout:Nothing to do. 2026-02-22T21:35:07.742 INFO:teuthology.orchestra.run.trial004.stdout:Complete! 2026-02-22T21:35:07.763 DEBUG:teuthology.orchestra.run.trial004:> sudo yum -y install cephadm 2026-02-22T21:35:08.076 INFO:teuthology.orchestra.run.trial004.stdout:Last metadata expiration check: 0:00:33 ago on Sun 22 Feb 2026 09:34:35 PM UTC. 2026-02-22T21:35:08.140 INFO:teuthology.orchestra.run.trial004.stdout:Package cephadm-2:19.2.3-689.gf90f4eae.el9.noarch is already installed. 2026-02-22T21:35:08.158 INFO:teuthology.orchestra.run.trial004.stdout:Dependencies resolved. 2026-02-22T21:35:08.158 INFO:teuthology.orchestra.run.trial004.stdout:Nothing to do. 2026-02-22T21:35:08.159 INFO:teuthology.orchestra.run.trial004.stdout:Complete! 2026-02-22T21:35:08.180 DEBUG:teuthology.orchestra.run.trial004:> sudo yum -y install ceph-immutable-object-cache 2026-02-22T21:35:08.490 INFO:teuthology.orchestra.run.trial004.stdout:Last metadata expiration check: 0:00:33 ago on Sun 22 Feb 2026 09:34:35 PM UTC. 2026-02-22T21:35:08.575 INFO:teuthology.orchestra.run.trial004.stdout:Dependencies resolved. 2026-02-22T21:35:08.575 INFO:teuthology.orchestra.run.trial004.stdout:================================================================================ 2026-02-22T21:35:08.575 INFO:teuthology.orchestra.run.trial004.stdout: Package Arch Version Repo Size 2026-02-22T21:35:08.576 INFO:teuthology.orchestra.run.trial004.stdout:================================================================================ 2026-02-22T21:35:08.576 INFO:teuthology.orchestra.run.trial004.stdout:Installing: 2026-02-22T21:35:08.576 INFO:teuthology.orchestra.run.trial004.stdout: ceph-immutable-object-cache x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 145 k 2026-02-22T21:35:08.576 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:35:08.576 INFO:teuthology.orchestra.run.trial004.stdout:Transaction Summary 2026-02-22T21:35:08.576 INFO:teuthology.orchestra.run.trial004.stdout:================================================================================ 2026-02-22T21:35:08.576 INFO:teuthology.orchestra.run.trial004.stdout:Install 1 Package 2026-02-22T21:35:08.576 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:35:08.576 INFO:teuthology.orchestra.run.trial004.stdout:Total download size: 145 k 2026-02-22T21:35:08.576 INFO:teuthology.orchestra.run.trial004.stdout:Installed size: 431 k 2026-02-22T21:35:08.576 INFO:teuthology.orchestra.run.trial004.stdout:Downloading Packages: 2026-02-22T21:35:08.662 INFO:teuthology.orchestra.run.trial004.stdout:ceph-immutable-object-cache-19.2.3-689.gf90f4ea 1.6 MB/s | 145 kB 00:00 2026-02-22T21:35:08.663 INFO:teuthology.orchestra.run.trial004.stdout:-------------------------------------------------------------------------------- 2026-02-22T21:35:08.663 INFO:teuthology.orchestra.run.trial004.stdout:Total 1.6 MB/s | 145 kB 00:00 2026-02-22T21:35:08.663 INFO:teuthology.orchestra.run.trial004.stdout:Running transaction check 2026-02-22T21:35:08.666 INFO:teuthology.orchestra.run.trial004.stdout:Transaction check succeeded. 2026-02-22T21:35:08.666 INFO:teuthology.orchestra.run.trial004.stdout:Running transaction test 2026-02-22T21:35:08.697 INFO:teuthology.orchestra.run.trial004.stdout:Transaction test succeeded. 2026-02-22T21:35:08.698 INFO:teuthology.orchestra.run.trial004.stdout:Running transaction 2026-02-22T21:35:08.752 INFO:teuthology.orchestra.run.trial004.stdout: Preparing : 1/1 2026-02-22T21:35:08.753 INFO:teuthology.orchestra.run.trial004.stdout: Installing : ceph-immutable-object-cache-2:19.2.3-689.gf90f4eae.e 1/1 2026-02-22T21:35:08.776 INFO:teuthology.orchestra.run.trial004.stdout: Running scriptlet: ceph-immutable-object-cache-2:19.2.3-689.gf90f4eae.e 1/1 2026-02-22T21:35:08.776 INFO:teuthology.orchestra.run.trial004.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-22T21:35:08.776 INFO:teuthology.orchestra.run.trial004.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2026-02-22T21:35:08.776 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:35:09.097 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : ceph-immutable-object-cache-2:19.2.3-689.gf90f4eae.e 1/1 2026-02-22T21:35:09.097 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:35:09.097 INFO:teuthology.orchestra.run.trial004.stdout:Installed: 2026-02-22T21:35:09.097 INFO:teuthology.orchestra.run.trial004.stdout: ceph-immutable-object-cache-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:35:09.097 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:35:09.097 INFO:teuthology.orchestra.run.trial004.stdout:Complete! 2026-02-22T21:35:09.123 DEBUG:teuthology.orchestra.run.trial004:> sudo yum -y install ceph-mgr 2026-02-22T21:35:09.430 INFO:teuthology.orchestra.run.trial004.stdout:Last metadata expiration check: 0:00:34 ago on Sun 22 Feb 2026 09:34:35 PM UTC. 2026-02-22T21:35:09.495 INFO:teuthology.orchestra.run.trial004.stdout:Package ceph-mgr-2:19.2.3-689.gf90f4eae.el9.x86_64 is already installed. 2026-02-22T21:35:09.513 INFO:teuthology.orchestra.run.trial004.stdout:Dependencies resolved. 2026-02-22T21:35:09.514 INFO:teuthology.orchestra.run.trial004.stdout:Nothing to do. 2026-02-22T21:35:09.514 INFO:teuthology.orchestra.run.trial004.stdout:Complete! 2026-02-22T21:35:09.535 DEBUG:teuthology.orchestra.run.trial004:> sudo yum -y install ceph-mgr-dashboard 2026-02-22T21:35:09.843 INFO:teuthology.orchestra.run.trial004.stdout:Last metadata expiration check: 0:00:34 ago on Sun 22 Feb 2026 09:34:35 PM UTC. 2026-02-22T21:35:09.908 INFO:teuthology.orchestra.run.trial004.stdout:Package ceph-mgr-dashboard-2:19.2.3-689.gf90f4eae.el9.noarch is already installed. 2026-02-22T21:35:09.926 INFO:teuthology.orchestra.run.trial004.stdout:Dependencies resolved. 2026-02-22T21:35:09.927 INFO:teuthology.orchestra.run.trial004.stdout:Nothing to do. 2026-02-22T21:35:09.927 INFO:teuthology.orchestra.run.trial004.stdout:Complete! 2026-02-22T21:35:09.948 DEBUG:teuthology.orchestra.run.trial004:> sudo yum -y install ceph-mgr-diskprediction-local 2026-02-22T21:35:10.254 INFO:teuthology.orchestra.run.trial004.stdout:Last metadata expiration check: 0:00:35 ago on Sun 22 Feb 2026 09:34:35 PM UTC. 2026-02-22T21:35:10.319 INFO:teuthology.orchestra.run.trial004.stdout:Package ceph-mgr-diskprediction-local-2:19.2.3-689.gf90f4eae.el9.noarch is already installed. 2026-02-22T21:35:10.337 INFO:teuthology.orchestra.run.trial004.stdout:Dependencies resolved. 2026-02-22T21:35:10.338 INFO:teuthology.orchestra.run.trial004.stdout:Nothing to do. 2026-02-22T21:35:10.338 INFO:teuthology.orchestra.run.trial004.stdout:Complete! 2026-02-22T21:35:10.358 DEBUG:teuthology.orchestra.run.trial004:> sudo yum -y install ceph-mgr-rook 2026-02-22T21:35:10.668 INFO:teuthology.orchestra.run.trial004.stdout:Last metadata expiration check: 0:00:35 ago on Sun 22 Feb 2026 09:34:35 PM UTC. 2026-02-22T21:35:10.734 INFO:teuthology.orchestra.run.trial004.stdout:Package ceph-mgr-rook-2:19.2.3-689.gf90f4eae.el9.noarch is already installed. 2026-02-22T21:35:10.752 INFO:teuthology.orchestra.run.trial004.stdout:Dependencies resolved. 2026-02-22T21:35:10.753 INFO:teuthology.orchestra.run.trial004.stdout:Nothing to do. 2026-02-22T21:35:10.753 INFO:teuthology.orchestra.run.trial004.stdout:Complete! 2026-02-22T21:35:10.775 DEBUG:teuthology.orchestra.run.trial004:> sudo yum -y install ceph-mgr-cephadm 2026-02-22T21:35:11.081 INFO:teuthology.orchestra.run.trial004.stdout:Last metadata expiration check: 0:00:36 ago on Sun 22 Feb 2026 09:34:35 PM UTC. 2026-02-22T21:35:11.145 INFO:teuthology.orchestra.run.trial004.stdout:Package ceph-mgr-cephadm-2:19.2.3-689.gf90f4eae.el9.noarch is already installed. 2026-02-22T21:35:11.163 INFO:teuthology.orchestra.run.trial004.stdout:Dependencies resolved. 2026-02-22T21:35:11.164 INFO:teuthology.orchestra.run.trial004.stdout:Nothing to do. 2026-02-22T21:35:11.164 INFO:teuthology.orchestra.run.trial004.stdout:Complete! 2026-02-22T21:35:11.184 DEBUG:teuthology.orchestra.run.trial004:> sudo yum -y install ceph-fuse 2026-02-22T21:35:11.468 INFO:teuthology.orchestra.run.trial026.stdout: Running scriptlet: ceph-selinux-2:19.2.3-689.gf90f4eae.el9.x86_64 34/37 2026-02-22T21:35:11.469 INFO:teuthology.orchestra.run.trial026.stdout:skipping the directory /sys 2026-02-22T21:35:11.469 INFO:teuthology.orchestra.run.trial026.stdout:skipping the directory /proc 2026-02-22T21:35:11.469 INFO:teuthology.orchestra.run.trial026.stdout:skipping the directory /mnt 2026-02-22T21:35:11.469 INFO:teuthology.orchestra.run.trial026.stdout:skipping the directory /var/tmp 2026-02-22T21:35:11.469 INFO:teuthology.orchestra.run.trial026.stdout:skipping the directory /home 2026-02-22T21:35:11.469 INFO:teuthology.orchestra.run.trial026.stdout:skipping the directory /root 2026-02-22T21:35:11.469 INFO:teuthology.orchestra.run.trial026.stdout:skipping the directory /tmp 2026-02-22T21:35:11.469 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:35:11.490 INFO:teuthology.orchestra.run.trial004.stdout:Last metadata expiration check: 0:00:36 ago on Sun 22 Feb 2026 09:34:35 PM UTC. 2026-02-22T21:35:11.573 INFO:teuthology.orchestra.run.trial004.stdout:Dependencies resolved. 2026-02-22T21:35:11.573 INFO:teuthology.orchestra.run.trial004.stdout:================================================================================ 2026-02-22T21:35:11.573 INFO:teuthology.orchestra.run.trial004.stdout: Package Architecture Version Repository Size 2026-02-22T21:35:11.573 INFO:teuthology.orchestra.run.trial004.stdout:================================================================================ 2026-02-22T21:35:11.574 INFO:teuthology.orchestra.run.trial004.stdout:Installing: 2026-02-22T21:35:11.574 INFO:teuthology.orchestra.run.trial004.stdout: ceph-fuse x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 1.2 M 2026-02-22T21:35:11.574 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:35:11.574 INFO:teuthology.orchestra.run.trial004.stdout:Transaction Summary 2026-02-22T21:35:11.574 INFO:teuthology.orchestra.run.trial004.stdout:================================================================================ 2026-02-22T21:35:11.574 INFO:teuthology.orchestra.run.trial004.stdout:Install 1 Package 2026-02-22T21:35:11.574 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:35:11.574 INFO:teuthology.orchestra.run.trial004.stdout:Total download size: 1.2 M 2026-02-22T21:35:11.574 INFO:teuthology.orchestra.run.trial004.stdout:Installed size: 3.6 M 2026-02-22T21:35:11.574 INFO:teuthology.orchestra.run.trial004.stdout:Downloading Packages: 2026-02-22T21:35:11.697 INFO:teuthology.orchestra.run.trial004.stdout:ceph-fuse-19.2.3-689.gf90f4eae.el9.x86_64.rpm 9.4 MB/s | 1.2 MB 00:00 2026-02-22T21:35:11.697 INFO:teuthology.orchestra.run.trial004.stdout:-------------------------------------------------------------------------------- 2026-02-22T21:35:11.697 INFO:teuthology.orchestra.run.trial004.stdout:Total 9.4 MB/s | 1.2 MB 00:00 2026-02-22T21:35:11.697 INFO:teuthology.orchestra.run.trial004.stdout:Running transaction check 2026-02-22T21:35:11.701 INFO:teuthology.orchestra.run.trial004.stdout:Transaction check succeeded. 2026-02-22T21:35:11.701 INFO:teuthology.orchestra.run.trial004.stdout:Running transaction test 2026-02-22T21:35:11.737 INFO:teuthology.orchestra.run.trial004.stdout:Transaction test succeeded. 2026-02-22T21:35:11.738 INFO:teuthology.orchestra.run.trial004.stdout:Running transaction 2026-02-22T21:35:11.841 INFO:teuthology.orchestra.run.trial004.stdout: Preparing : 1/1 2026-02-22T21:35:11.858 INFO:teuthology.orchestra.run.trial004.stdout: Installing : ceph-fuse-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T21:35:11.918 INFO:teuthology.orchestra.run.trial026.stdout: Installing : ceph-radosgw-2:19.2.3-689.gf90f4eae.el9.x86_64 35/37 2026-02-22T21:35:11.941 INFO:teuthology.orchestra.run.trial026.stdout: Running scriptlet: ceph-radosgw-2:19.2.3-689.gf90f4eae.el9.x86_64 35/37 2026-02-22T21:35:11.941 INFO:teuthology.orchestra.run.trial026.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-22T21:35:11.941 INFO:teuthology.orchestra.run.trial026.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2026-02-22T21:35:11.942 INFO:teuthology.orchestra.run.trial026.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2026-02-22T21:35:11.942 INFO:teuthology.orchestra.run.trial026.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2026-02-22T21:35:11.942 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:35:11.943 INFO:teuthology.orchestra.run.trial026.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 36/37 2026-02-22T21:35:11.955 INFO:teuthology.orchestra.run.trial026.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 36/37 2026-02-22T21:35:11.955 INFO:teuthology.orchestra.run.trial026.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 37/37 2026-02-22T21:35:12.010 INFO:teuthology.orchestra.run.trial026.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 37/37 2026-02-22T21:35:12.011 INFO:teuthology.orchestra.run.trial026.stdout:Creating group 'qat' with GID 992. 2026-02-22T21:35:12.011 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:35:12.080 INFO:teuthology.orchestra.run.trial004.stdout: Running scriptlet: ceph-fuse-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T21:35:12.136 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : ceph-fuse-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T21:35:12.137 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:35:12.137 INFO:teuthology.orchestra.run.trial004.stdout:Installed: 2026-02-22T21:35:12.137 INFO:teuthology.orchestra.run.trial004.stdout: ceph-fuse-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:35:12.137 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:35:12.137 INFO:teuthology.orchestra.run.trial004.stdout:Complete! 2026-02-22T21:35:12.165 DEBUG:teuthology.orchestra.run.trial004:> sudo yum -y install ceph-volume 2026-02-22T21:35:12.290 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : ceph-base-2:19.2.3-689.gf90f4eae.el9.x86_64 1/37 2026-02-22T21:35:12.290 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : ceph-common-2:19.2.3-689.gf90f4eae.el9.x86_64 2/37 2026-02-22T21:35:12.290 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : ceph-radosgw-2:19.2.3-689.gf90f4eae.el9.x86_64 3/37 2026-02-22T21:35:12.290 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : ceph-selinux-2:19.2.3-689.gf90f4eae.el9.x86_64 4/37 2026-02-22T21:35:12.291 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : libcephfs2-2:19.2.3-689.gf90f4eae.el9.x86_64 5/37 2026-02-22T21:35:12.291 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : libradosstriper1-2:19.2.3-689.gf90f4eae.el9.x86_64 6/37 2026-02-22T21:35:12.291 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : librgw2-2:19.2.3-689.gf90f4eae.el9.x86_64 7/37 2026-02-22T21:35:12.291 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : python3-ceph-argparse-2:19.2.3-689.gf90f4eae.el9.x 8/37 2026-02-22T21:35:12.291 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : python3-ceph-common-2:19.2.3-689.gf90f4eae.el9.x86 9/37 2026-02-22T21:35:12.291 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : python3-cephfs-2:19.2.3-689.gf90f4eae.el9.x86_64 10/37 2026-02-22T21:35:12.291 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : python3-rados-2:19.2.3-689.gf90f4eae.el9.x86_64 11/37 2026-02-22T21:35:12.291 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : python3-rbd-2:19.2.3-689.gf90f4eae.el9.x86_64 12/37 2026-02-22T21:35:12.291 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : python3-rgw-2:19.2.3-689.gf90f4eae.el9.x86_64 13/37 2026-02-22T21:35:12.291 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : mailcap-2.1.49-5.el9.noarch 14/37 2026-02-22T21:35:12.291 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : nvme-cli-2.16-1.el9.x86_64 15/37 2026-02-22T21:35:12.291 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : pciutils-3.7.0-7.el9.x86_64 16/37 2026-02-22T21:35:12.291 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : boost-program-options-1.75.0-13.el9.x86_64 17/37 2026-02-22T21:35:12.291 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : libnbd-1.20.3-4.el9.x86_64 18/37 2026-02-22T21:35:12.291 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 19/37 2026-02-22T21:35:12.292 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 20/37 2026-02-22T21:35:12.292 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 21/37 2026-02-22T21:35:12.292 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 22/37 2026-02-22T21:35:12.292 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : qatlib-25.08.0-2.el9.x86_64 23/37 2026-02-22T21:35:12.292 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : qatlib-service-25.08.0-2.el9.x86_64 24/37 2026-02-22T21:35:12.292 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : qatzip-libs-1.3.1-1.el9.x86_64 25/37 2026-02-22T21:35:12.292 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : gperftools-libs-2.9.1-3.el9.x86_64 26/37 2026-02-22T21:35:12.292 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : libarrow-9.0.0-15.el9.x86_64 27/37 2026-02-22T21:35:12.292 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : libarrow-doc-9.0.0-15.el9.noarch 28/37 2026-02-22T21:35:12.292 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : liboath-2.6.12-1.el9.x86_64 29/37 2026-02-22T21:35:12.292 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 30/37 2026-02-22T21:35:12.292 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : parquet-libs-9.0.0-15.el9.x86_64 31/37 2026-02-22T21:35:12.292 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : re2-1:20211101-20.el9.x86_64 32/37 2026-02-22T21:35:12.292 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : thrift-0.15.0-4.el9.x86_64 33/37 2026-02-22T21:35:12.292 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : librados2-2:19.2.3-689.gf90f4eae.el9.x86_64 34/37 2026-02-22T21:35:12.293 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 35/37 2026-02-22T21:35:12.293 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : librbd1-2:19.2.3-689.gf90f4eae.el9.x86_64 36/37 2026-02-22T21:35:12.352 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 37/37 2026-02-22T21:35:12.352 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:35:12.353 INFO:teuthology.orchestra.run.trial026.stdout:Upgraded: 2026-02-22T21:35:12.353 INFO:teuthology.orchestra.run.trial026.stdout: librados2-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:35:12.353 INFO:teuthology.orchestra.run.trial026.stdout: librbd1-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:35:12.353 INFO:teuthology.orchestra.run.trial026.stdout:Installed: 2026-02-22T21:35:12.353 INFO:teuthology.orchestra.run.trial026.stdout: boost-program-options-1.75.0-13.el9.x86_64 2026-02-22T21:35:12.353 INFO:teuthology.orchestra.run.trial026.stdout: ceph-base-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:35:12.353 INFO:teuthology.orchestra.run.trial026.stdout: ceph-common-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:35:12.353 INFO:teuthology.orchestra.run.trial026.stdout: ceph-radosgw-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:35:12.353 INFO:teuthology.orchestra.run.trial026.stdout: ceph-selinux-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:35:12.353 INFO:teuthology.orchestra.run.trial026.stdout: gperftools-libs-2.9.1-3.el9.x86_64 2026-02-22T21:35:12.354 INFO:teuthology.orchestra.run.trial026.stdout: libarrow-9.0.0-15.el9.x86_64 2026-02-22T21:35:12.354 INFO:teuthology.orchestra.run.trial026.stdout: libarrow-doc-9.0.0-15.el9.noarch 2026-02-22T21:35:12.354 INFO:teuthology.orchestra.run.trial026.stdout: libcephfs2-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:35:12.354 INFO:teuthology.orchestra.run.trial026.stdout: libnbd-1.20.3-4.el9.x86_64 2026-02-22T21:35:12.354 INFO:teuthology.orchestra.run.trial026.stdout: liboath-2.6.12-1.el9.x86_64 2026-02-22T21:35:12.354 INFO:teuthology.orchestra.run.trial026.stdout: libpmemobj-1.12.1-1.el9.x86_64 2026-02-22T21:35:12.354 INFO:teuthology.orchestra.run.trial026.stdout: librabbitmq-0.11.0-7.el9.x86_64 2026-02-22T21:35:12.354 INFO:teuthology.orchestra.run.trial026.stdout: libradosstriper1-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:35:12.354 INFO:teuthology.orchestra.run.trial026.stdout: librdkafka-1.6.1-102.el9.x86_64 2026-02-22T21:35:12.354 INFO:teuthology.orchestra.run.trial026.stdout: librgw2-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:35:12.354 INFO:teuthology.orchestra.run.trial026.stdout: libunwind-1.6.2-1.el9.x86_64 2026-02-22T21:35:12.354 INFO:teuthology.orchestra.run.trial026.stdout: lttng-ust-2.12.0-6.el9.x86_64 2026-02-22T21:35:12.355 INFO:teuthology.orchestra.run.trial026.stdout: mailcap-2.1.49-5.el9.noarch 2026-02-22T21:35:12.355 INFO:teuthology.orchestra.run.trial026.stdout: nvme-cli-2.16-1.el9.x86_64 2026-02-22T21:35:12.355 INFO:teuthology.orchestra.run.trial026.stdout: parquet-libs-9.0.0-15.el9.x86_64 2026-02-22T21:35:12.355 INFO:teuthology.orchestra.run.trial026.stdout: pciutils-3.7.0-7.el9.x86_64 2026-02-22T21:35:12.355 INFO:teuthology.orchestra.run.trial026.stdout: python3-ceph-argparse-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:35:12.355 INFO:teuthology.orchestra.run.trial026.stdout: python3-ceph-common-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:35:12.355 INFO:teuthology.orchestra.run.trial026.stdout: python3-cephfs-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:35:12.355 INFO:teuthology.orchestra.run.trial026.stdout: python3-rados-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:35:12.355 INFO:teuthology.orchestra.run.trial026.stdout: python3-rbd-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:35:12.355 INFO:teuthology.orchestra.run.trial026.stdout: python3-rgw-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:35:12.355 INFO:teuthology.orchestra.run.trial026.stdout: qatlib-25.08.0-2.el9.x86_64 2026-02-22T21:35:12.355 INFO:teuthology.orchestra.run.trial026.stdout: qatlib-service-25.08.0-2.el9.x86_64 2026-02-22T21:35:12.356 INFO:teuthology.orchestra.run.trial026.stdout: qatzip-libs-1.3.1-1.el9.x86_64 2026-02-22T21:35:12.356 INFO:teuthology.orchestra.run.trial026.stdout: re2-1:20211101-20.el9.x86_64 2026-02-22T21:35:12.356 INFO:teuthology.orchestra.run.trial026.stdout: thrift-0.15.0-4.el9.x86_64 2026-02-22T21:35:12.356 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:35:12.356 INFO:teuthology.orchestra.run.trial026.stdout:Complete! 2026-02-22T21:35:12.392 DEBUG:teuthology.orchestra.run.trial026:> sudo yum -y install ceph-test 2026-02-22T21:35:12.475 INFO:teuthology.orchestra.run.trial004.stdout:Last metadata expiration check: 0:00:37 ago on Sun 22 Feb 2026 09:34:35 PM UTC. 2026-02-22T21:35:12.540 INFO:teuthology.orchestra.run.trial004.stdout:Package ceph-volume-2:19.2.3-689.gf90f4eae.el9.noarch is already installed. 2026-02-22T21:35:12.558 INFO:teuthology.orchestra.run.trial004.stdout:Dependencies resolved. 2026-02-22T21:35:12.559 INFO:teuthology.orchestra.run.trial004.stdout:Nothing to do. 2026-02-22T21:35:12.559 INFO:teuthology.orchestra.run.trial004.stdout:Complete! 2026-02-22T21:35:12.580 DEBUG:teuthology.orchestra.run.trial004:> sudo yum -y install librados-devel 2026-02-22T21:35:12.700 INFO:teuthology.orchestra.run.trial026.stdout:Last metadata expiration check: 0:00:13 ago on Sun 22 Feb 2026 09:34:59 PM UTC. 2026-02-22T21:35:12.785 INFO:teuthology.orchestra.run.trial026.stdout:Dependencies resolved. 2026-02-22T21:35:12.786 INFO:teuthology.orchestra.run.trial026.stdout:================================================================================ 2026-02-22T21:35:12.786 INFO:teuthology.orchestra.run.trial026.stdout: Package Arch Version Repository Size 2026-02-22T21:35:12.786 INFO:teuthology.orchestra.run.trial026.stdout:================================================================================ 2026-02-22T21:35:12.787 INFO:teuthology.orchestra.run.trial026.stdout:Installing: 2026-02-22T21:35:12.787 INFO:teuthology.orchestra.run.trial026.stdout: ceph-test x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 50 M 2026-02-22T21:35:12.787 INFO:teuthology.orchestra.run.trial026.stdout:Installing dependencies: 2026-02-22T21:35:12.787 INFO:teuthology.orchestra.run.trial026.stdout: libcephsqlite x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 163 k 2026-02-22T21:35:12.787 INFO:teuthology.orchestra.run.trial026.stdout: libxslt x86_64 1.1.34-12.el9 appstream 233 k 2026-02-22T21:35:12.787 INFO:teuthology.orchestra.run.trial026.stdout: socat x86_64 1.7.4.1-8.el9 appstream 303 k 2026-02-22T21:35:12.787 INFO:teuthology.orchestra.run.trial026.stdout: xmlstarlet x86_64 1.6.1-20.el9 appstream 64 k 2026-02-22T21:35:12.787 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:35:12.787 INFO:teuthology.orchestra.run.trial026.stdout:Transaction Summary 2026-02-22T21:35:12.787 INFO:teuthology.orchestra.run.trial026.stdout:================================================================================ 2026-02-22T21:35:12.787 INFO:teuthology.orchestra.run.trial026.stdout:Install 5 Packages 2026-02-22T21:35:12.788 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:35:12.788 INFO:teuthology.orchestra.run.trial026.stdout:Total download size: 50 M 2026-02-22T21:35:12.788 INFO:teuthology.orchestra.run.trial026.stdout:Installed size: 212 M 2026-02-22T21:35:12.788 INFO:teuthology.orchestra.run.trial026.stdout:Downloading Packages: 2026-02-22T21:35:12.887 INFO:teuthology.orchestra.run.trial004.stdout:Last metadata expiration check: 0:00:37 ago on Sun 22 Feb 2026 09:34:35 PM UTC. 2026-02-22T21:35:12.971 INFO:teuthology.orchestra.run.trial004.stdout:Dependencies resolved. 2026-02-22T21:35:12.971 INFO:teuthology.orchestra.run.trial004.stdout:================================================================================ 2026-02-22T21:35:12.971 INFO:teuthology.orchestra.run.trial004.stdout: Package Arch Version Repo Size 2026-02-22T21:35:12.971 INFO:teuthology.orchestra.run.trial004.stdout:================================================================================ 2026-02-22T21:35:12.971 INFO:teuthology.orchestra.run.trial004.stdout:Installing: 2026-02-22T21:35:12.971 INFO:teuthology.orchestra.run.trial004.stdout: librados-devel x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 126 k 2026-02-22T21:35:12.971 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:35:12.972 INFO:teuthology.orchestra.run.trial004.stdout:Transaction Summary 2026-02-22T21:35:12.972 INFO:teuthology.orchestra.run.trial004.stdout:================================================================================ 2026-02-22T21:35:12.972 INFO:teuthology.orchestra.run.trial004.stdout:Install 1 Package 2026-02-22T21:35:12.972 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:35:12.972 INFO:teuthology.orchestra.run.trial004.stdout:Total download size: 126 k 2026-02-22T21:35:12.972 INFO:teuthology.orchestra.run.trial004.stdout:Installed size: 456 k 2026-02-22T21:35:12.972 INFO:teuthology.orchestra.run.trial004.stdout:Downloading Packages: 2026-02-22T21:35:13.338 INFO:teuthology.orchestra.run.trial004.stdout:librados-devel-19.2.3-689.gf90f4eae.el9.x86_64. 345 kB/s | 126 kB 00:00 2026-02-22T21:35:13.339 INFO:teuthology.orchestra.run.trial004.stdout:-------------------------------------------------------------------------------- 2026-02-22T21:35:13.339 INFO:teuthology.orchestra.run.trial004.stdout:Total 344 kB/s | 126 kB 00:00 2026-02-22T21:35:13.339 INFO:teuthology.orchestra.run.trial004.stdout:Running transaction check 2026-02-22T21:35:13.341 INFO:teuthology.orchestra.run.trial004.stdout:Transaction check succeeded. 2026-02-22T21:35:13.341 INFO:teuthology.orchestra.run.trial004.stdout:Running transaction test 2026-02-22T21:35:13.374 INFO:teuthology.orchestra.run.trial004.stdout:Transaction test succeeded. 2026-02-22T21:35:13.374 INFO:teuthology.orchestra.run.trial004.stdout:Running transaction 2026-02-22T21:35:13.418 INFO:teuthology.orchestra.run.trial026.stdout:(1/5): libcephsqlite-19.2.3-689.gf90f4eae.el9.x 443 kB/s | 163 kB 00:00 2026-02-22T21:35:13.428 INFO:teuthology.orchestra.run.trial004.stdout: Preparing : 1/1 2026-02-22T21:35:13.445 INFO:teuthology.orchestra.run.trial004.stdout: Installing : librados-devel-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T21:35:13.527 INFO:teuthology.orchestra.run.trial004.stdout: Running scriptlet: librados-devel-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T21:35:13.585 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : librados-devel-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T21:35:13.585 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:35:13.585 INFO:teuthology.orchestra.run.trial004.stdout:Installed: 2026-02-22T21:35:13.585 INFO:teuthology.orchestra.run.trial004.stdout: librados-devel-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:35:13.586 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:35:13.586 INFO:teuthology.orchestra.run.trial004.stdout:Complete! 2026-02-22T21:35:13.613 DEBUG:teuthology.orchestra.run.trial004:> sudo yum -y install libcephfs2 2026-02-22T21:35:13.927 INFO:teuthology.orchestra.run.trial004.stdout:Last metadata expiration check: 0:00:38 ago on Sun 22 Feb 2026 09:34:35 PM UTC. 2026-02-22T21:35:13.992 INFO:teuthology.orchestra.run.trial004.stdout:Package libcephfs2-2:19.2.3-689.gf90f4eae.el9.x86_64 is already installed. 2026-02-22T21:35:14.010 INFO:teuthology.orchestra.run.trial004.stdout:Dependencies resolved. 2026-02-22T21:35:14.011 INFO:teuthology.orchestra.run.trial004.stdout:Nothing to do. 2026-02-22T21:35:14.011 INFO:teuthology.orchestra.run.trial004.stdout:Complete! 2026-02-22T21:35:14.031 DEBUG:teuthology.orchestra.run.trial004:> sudo yum -y install libcephfs-devel 2026-02-22T21:35:14.223 INFO:teuthology.orchestra.run.trial026.stdout:(2/5): socat-1.7.4.1-8.el9.x86_64.rpm 376 kB/s | 303 kB 00:00 2026-02-22T21:35:14.263 INFO:teuthology.orchestra.run.trial026.stdout:(3/5): libxslt-1.1.34-12.el9.x86_64.rpm 192 kB/s | 233 kB 00:01 2026-02-22T21:35:14.322 INFO:teuthology.orchestra.run.trial026.stdout:(4/5): xmlstarlet-1.6.1-20.el9.x86_64.rpm 646 kB/s | 64 kB 00:00 2026-02-22T21:35:14.338 INFO:teuthology.orchestra.run.trial004.stdout:Last metadata expiration check: 0:00:39 ago on Sun 22 Feb 2026 09:34:35 PM UTC. 2026-02-22T21:35:14.421 INFO:teuthology.orchestra.run.trial004.stdout:Dependencies resolved. 2026-02-22T21:35:14.422 INFO:teuthology.orchestra.run.trial004.stdout:================================================================================ 2026-02-22T21:35:14.422 INFO:teuthology.orchestra.run.trial004.stdout: Package Arch Version Repo Size 2026-02-22T21:35:14.422 INFO:teuthology.orchestra.run.trial004.stdout:================================================================================ 2026-02-22T21:35:14.422 INFO:teuthology.orchestra.run.trial004.stdout:Installing: 2026-02-22T21:35:14.422 INFO:teuthology.orchestra.run.trial004.stdout: libcephfs-devel x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 34 k 2026-02-22T21:35:14.422 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:35:14.422 INFO:teuthology.orchestra.run.trial004.stdout:Transaction Summary 2026-02-22T21:35:14.422 INFO:teuthology.orchestra.run.trial004.stdout:================================================================================ 2026-02-22T21:35:14.422 INFO:teuthology.orchestra.run.trial004.stdout:Install 1 Package 2026-02-22T21:35:14.423 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:35:14.423 INFO:teuthology.orchestra.run.trial004.stdout:Total download size: 34 k 2026-02-22T21:35:14.423 INFO:teuthology.orchestra.run.trial004.stdout:Installed size: 153 k 2026-02-22T21:35:14.423 INFO:teuthology.orchestra.run.trial004.stdout:Downloading Packages: 2026-02-22T21:35:14.556 INFO:teuthology.orchestra.run.trial004.stdout:libcephfs-devel-19.2.3-689.gf90f4eae.el9.x86_64 251 kB/s | 34 kB 00:00 2026-02-22T21:35:14.557 INFO:teuthology.orchestra.run.trial004.stdout:-------------------------------------------------------------------------------- 2026-02-22T21:35:14.557 INFO:teuthology.orchestra.run.trial004.stdout:Total 249 kB/s | 34 kB 00:00 2026-02-22T21:35:14.557 INFO:teuthology.orchestra.run.trial004.stdout:Running transaction check 2026-02-22T21:35:14.559 INFO:teuthology.orchestra.run.trial004.stdout:Transaction check succeeded. 2026-02-22T21:35:14.559 INFO:teuthology.orchestra.run.trial004.stdout:Running transaction test 2026-02-22T21:35:14.562 INFO:teuthology.orchestra.run.trial004.stdout:Transaction test succeeded. 2026-02-22T21:35:14.562 INFO:teuthology.orchestra.run.trial004.stdout:Running transaction 2026-02-22T21:35:14.583 INFO:teuthology.orchestra.run.trial004.stdout: Preparing : 1/1 2026-02-22T21:35:14.599 INFO:teuthology.orchestra.run.trial004.stdout: Installing : libcephfs-devel-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T21:35:14.600 INFO:teuthology.orchestra.run.trial026.stdout:(5/5): ceph-test-19.2.3-689.gf90f4eae.el9.x86_6 32 MB/s | 50 MB 00:01 2026-02-22T21:35:14.602 INFO:teuthology.orchestra.run.trial026.stdout:-------------------------------------------------------------------------------- 2026-02-22T21:35:14.602 INFO:teuthology.orchestra.run.trial026.stdout:Total 28 MB/s | 50 MB 00:01 2026-02-22T21:35:14.629 INFO:teuthology.orchestra.run.trial026.stdout:Running transaction check 2026-02-22T21:35:14.640 INFO:teuthology.orchestra.run.trial026.stdout:Transaction check succeeded. 2026-02-22T21:35:14.640 INFO:teuthology.orchestra.run.trial026.stdout:Running transaction test 2026-02-22T21:35:14.658 INFO:teuthology.orchestra.run.trial004.stdout: Running scriptlet: libcephfs-devel-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T21:35:14.719 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : libcephfs-devel-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T21:35:14.719 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:35:14.719 INFO:teuthology.orchestra.run.trial004.stdout:Installed: 2026-02-22T21:35:14.719 INFO:teuthology.orchestra.run.trial004.stdout: libcephfs-devel-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:35:14.719 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:35:14.719 INFO:teuthology.orchestra.run.trial004.stdout:Complete! 2026-02-22T21:35:14.746 DEBUG:teuthology.orchestra.run.trial004:> sudo yum -y install librados2 2026-02-22T21:35:14.827 INFO:teuthology.orchestra.run.trial026.stdout:Transaction test succeeded. 2026-02-22T21:35:14.827 INFO:teuthology.orchestra.run.trial026.stdout:Running transaction 2026-02-22T21:35:15.038 INFO:teuthology.orchestra.run.trial026.stdout: Preparing : 1/1 2026-02-22T21:35:15.044 INFO:teuthology.orchestra.run.trial026.stdout: Installing : socat-1.7.4.1-8.el9.x86_64 1/5 2026-02-22T21:35:15.048 INFO:teuthology.orchestra.run.trial026.stdout: Installing : libxslt-1.1.34-12.el9.x86_64 2/5 2026-02-22T21:35:15.056 INFO:teuthology.orchestra.run.trial026.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 3/5 2026-02-22T21:35:15.057 INFO:teuthology.orchestra.run.trial004.stdout:Last metadata expiration check: 0:00:40 ago on Sun 22 Feb 2026 09:34:35 PM UTC. 2026-02-22T21:35:15.057 INFO:teuthology.orchestra.run.trial026.stdout: Installing : libcephsqlite-2:19.2.3-689.gf90f4eae.el9.x86_64 4/5 2026-02-22T21:35:15.121 INFO:teuthology.orchestra.run.trial004.stdout:Package librados2-2:19.2.3-689.gf90f4eae.el9.x86_64 is already installed. 2026-02-22T21:35:15.139 INFO:teuthology.orchestra.run.trial004.stdout:Dependencies resolved. 2026-02-22T21:35:15.140 INFO:teuthology.orchestra.run.trial004.stdout:Nothing to do. 2026-02-22T21:35:15.140 INFO:teuthology.orchestra.run.trial004.stdout:Complete! 2026-02-22T21:35:15.162 DEBUG:teuthology.orchestra.run.trial004:> sudo yum -y install librbd1 2026-02-22T21:35:15.467 INFO:teuthology.orchestra.run.trial004.stdout:Last metadata expiration check: 0:00:40 ago on Sun 22 Feb 2026 09:34:35 PM UTC. 2026-02-22T21:35:15.532 INFO:teuthology.orchestra.run.trial004.stdout:Package librbd1-2:19.2.3-689.gf90f4eae.el9.x86_64 is already installed. 2026-02-22T21:35:15.550 INFO:teuthology.orchestra.run.trial004.stdout:Dependencies resolved. 2026-02-22T21:35:15.550 INFO:teuthology.orchestra.run.trial004.stdout:Nothing to do. 2026-02-22T21:35:15.550 INFO:teuthology.orchestra.run.trial004.stdout:Complete! 2026-02-22T21:35:15.572 DEBUG:teuthology.orchestra.run.trial004:> sudo yum -y install python3-rados 2026-02-22T21:35:15.883 INFO:teuthology.orchestra.run.trial004.stdout:Last metadata expiration check: 0:00:40 ago on Sun 22 Feb 2026 09:34:35 PM UTC. 2026-02-22T21:35:15.948 INFO:teuthology.orchestra.run.trial004.stdout:Package python3-rados-2:19.2.3-689.gf90f4eae.el9.x86_64 is already installed. 2026-02-22T21:35:15.966 INFO:teuthology.orchestra.run.trial004.stdout:Dependencies resolved. 2026-02-22T21:35:15.966 INFO:teuthology.orchestra.run.trial004.stdout:Nothing to do. 2026-02-22T21:35:15.966 INFO:teuthology.orchestra.run.trial004.stdout:Complete! 2026-02-22T21:35:15.986 DEBUG:teuthology.orchestra.run.trial004:> sudo yum -y install python3-rgw 2026-02-22T21:35:16.300 INFO:teuthology.orchestra.run.trial004.stdout:Last metadata expiration check: 0:00:41 ago on Sun 22 Feb 2026 09:34:35 PM UTC. 2026-02-22T21:35:16.365 INFO:teuthology.orchestra.run.trial004.stdout:Package python3-rgw-2:19.2.3-689.gf90f4eae.el9.x86_64 is already installed. 2026-02-22T21:35:16.383 INFO:teuthology.orchestra.run.trial004.stdout:Dependencies resolved. 2026-02-22T21:35:16.383 INFO:teuthology.orchestra.run.trial004.stdout:Nothing to do. 2026-02-22T21:35:16.383 INFO:teuthology.orchestra.run.trial004.stdout:Complete! 2026-02-22T21:35:16.405 DEBUG:teuthology.orchestra.run.trial004:> sudo yum -y install python3-cephfs 2026-02-22T21:35:16.709 INFO:teuthology.orchestra.run.trial004.stdout:Last metadata expiration check: 0:00:41 ago on Sun 22 Feb 2026 09:34:35 PM UTC. 2026-02-22T21:35:16.775 INFO:teuthology.orchestra.run.trial004.stdout:Package python3-cephfs-2:19.2.3-689.gf90f4eae.el9.x86_64 is already installed. 2026-02-22T21:35:16.793 INFO:teuthology.orchestra.run.trial004.stdout:Dependencies resolved. 2026-02-22T21:35:16.794 INFO:teuthology.orchestra.run.trial004.stdout:Nothing to do. 2026-02-22T21:35:16.794 INFO:teuthology.orchestra.run.trial004.stdout:Complete! 2026-02-22T21:35:16.814 DEBUG:teuthology.orchestra.run.trial004:> sudo yum -y install python3-rbd 2026-02-22T21:35:17.125 INFO:teuthology.orchestra.run.trial004.stdout:Last metadata expiration check: 0:00:42 ago on Sun 22 Feb 2026 09:34:35 PM UTC. 2026-02-22T21:35:17.189 INFO:teuthology.orchestra.run.trial004.stdout:Package python3-rbd-2:19.2.3-689.gf90f4eae.el9.x86_64 is already installed. 2026-02-22T21:35:17.208 INFO:teuthology.orchestra.run.trial004.stdout:Dependencies resolved. 2026-02-22T21:35:17.208 INFO:teuthology.orchestra.run.trial004.stdout:Nothing to do. 2026-02-22T21:35:17.208 INFO:teuthology.orchestra.run.trial004.stdout:Complete! 2026-02-22T21:35:17.230 DEBUG:teuthology.orchestra.run.trial004:> sudo yum -y install rbd-fuse 2026-02-22T21:35:17.387 INFO:teuthology.orchestra.run.trial026.stdout: Running scriptlet: libcephsqlite-2:19.2.3-689.gf90f4eae.el9.x86_64 4/5 2026-02-22T21:35:17.406 INFO:teuthology.orchestra.run.trial026.stdout: Installing : ceph-test-2:19.2.3-689.gf90f4eae.el9.x86_64 5/5 2026-02-22T21:35:17.537 INFO:teuthology.orchestra.run.trial004.stdout:Last metadata expiration check: 0:00:42 ago on Sun 22 Feb 2026 09:34:35 PM UTC. 2026-02-22T21:35:17.620 INFO:teuthology.orchestra.run.trial004.stdout:Dependencies resolved. 2026-02-22T21:35:17.621 INFO:teuthology.orchestra.run.trial004.stdout:================================================================================ 2026-02-22T21:35:17.621 INFO:teuthology.orchestra.run.trial004.stdout: Package Architecture Version Repository Size 2026-02-22T21:35:17.621 INFO:teuthology.orchestra.run.trial004.stdout:================================================================================ 2026-02-22T21:35:17.621 INFO:teuthology.orchestra.run.trial004.stdout:Installing: 2026-02-22T21:35:17.621 INFO:teuthology.orchestra.run.trial004.stdout: rbd-fuse x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 85 k 2026-02-22T21:35:17.621 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:35:17.621 INFO:teuthology.orchestra.run.trial004.stdout:Transaction Summary 2026-02-22T21:35:17.621 INFO:teuthology.orchestra.run.trial004.stdout:================================================================================ 2026-02-22T21:35:17.622 INFO:teuthology.orchestra.run.trial004.stdout:Install 1 Package 2026-02-22T21:35:17.622 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:35:17.622 INFO:teuthology.orchestra.run.trial004.stdout:Total download size: 85 k 2026-02-22T21:35:17.622 INFO:teuthology.orchestra.run.trial004.stdout:Installed size: 227 k 2026-02-22T21:35:17.622 INFO:teuthology.orchestra.run.trial004.stdout:Downloading Packages: 2026-02-22T21:35:17.641 INFO:teuthology.orchestra.run.trial026.stdout: Running scriptlet: ceph-test-2:19.2.3-689.gf90f4eae.el9.x86_64 5/5 2026-02-22T21:35:17.641 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : ceph-test-2:19.2.3-689.gf90f4eae.el9.x86_64 1/5 2026-02-22T21:35:17.641 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : libcephsqlite-2:19.2.3-689.gf90f4eae.el9.x86_64 2/5 2026-02-22T21:35:17.642 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : libxslt-1.1.34-12.el9.x86_64 3/5 2026-02-22T21:35:17.642 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : socat-1.7.4.1-8.el9.x86_64 4/5 2026-02-22T21:35:17.695 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 5/5 2026-02-22T21:35:17.696 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:35:17.696 INFO:teuthology.orchestra.run.trial026.stdout:Installed: 2026-02-22T21:35:17.696 INFO:teuthology.orchestra.run.trial026.stdout: ceph-test-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:35:17.696 INFO:teuthology.orchestra.run.trial026.stdout: libcephsqlite-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:35:17.696 INFO:teuthology.orchestra.run.trial026.stdout: libxslt-1.1.34-12.el9.x86_64 2026-02-22T21:35:17.696 INFO:teuthology.orchestra.run.trial026.stdout: socat-1.7.4.1-8.el9.x86_64 2026-02-22T21:35:17.696 INFO:teuthology.orchestra.run.trial026.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2026-02-22T21:35:17.696 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:35:17.696 INFO:teuthology.orchestra.run.trial026.stdout:Complete! 2026-02-22T21:35:17.729 DEBUG:teuthology.orchestra.run.trial026:> sudo yum -y install ceph 2026-02-22T21:35:17.849 INFO:teuthology.orchestra.run.trial004.stdout:rbd-fuse-19.2.3-689.gf90f4eae.el9.x86_64.rpm 374 kB/s | 85 kB 00:00 2026-02-22T21:35:17.849 INFO:teuthology.orchestra.run.trial004.stdout:-------------------------------------------------------------------------------- 2026-02-22T21:35:17.850 INFO:teuthology.orchestra.run.trial004.stdout:Total 372 kB/s | 85 kB 00:00 2026-02-22T21:35:17.850 INFO:teuthology.orchestra.run.trial004.stdout:Running transaction check 2026-02-22T21:35:17.853 INFO:teuthology.orchestra.run.trial004.stdout:Transaction check succeeded. 2026-02-22T21:35:17.853 INFO:teuthology.orchestra.run.trial004.stdout:Running transaction test 2026-02-22T21:35:17.884 INFO:teuthology.orchestra.run.trial004.stdout:Transaction test succeeded. 2026-02-22T21:35:17.884 INFO:teuthology.orchestra.run.trial004.stdout:Running transaction 2026-02-22T21:35:17.935 INFO:teuthology.orchestra.run.trial004.stdout: Preparing : 1/1 2026-02-22T21:35:17.952 INFO:teuthology.orchestra.run.trial004.stdout: Installing : rbd-fuse-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T21:35:18.030 INFO:teuthology.orchestra.run.trial004.stdout: Running scriptlet: rbd-fuse-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T21:35:18.044 INFO:teuthology.orchestra.run.trial026.stdout:Last metadata expiration check: 0:00:19 ago on Sun 22 Feb 2026 09:34:59 PM UTC. 2026-02-22T21:35:18.087 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : rbd-fuse-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T21:35:18.087 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:35:18.087 INFO:teuthology.orchestra.run.trial004.stdout:Installed: 2026-02-22T21:35:18.087 INFO:teuthology.orchestra.run.trial004.stdout: rbd-fuse-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:35:18.087 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:35:18.087 INFO:teuthology.orchestra.run.trial004.stdout:Complete! 2026-02-22T21:35:18.113 DEBUG:teuthology.orchestra.run.trial004:> sudo yum -y install rbd-mirror 2026-02-22T21:35:18.132 INFO:teuthology.orchestra.run.trial026.stdout:Dependencies resolved. 2026-02-22T21:35:18.135 INFO:teuthology.orchestra.run.trial026.stdout:====================================================================================== 2026-02-22T21:35:18.135 INFO:teuthology.orchestra.run.trial026.stdout: Package Arch Version Repository Size 2026-02-22T21:35:18.135 INFO:teuthology.orchestra.run.trial026.stdout:====================================================================================== 2026-02-22T21:35:18.135 INFO:teuthology.orchestra.run.trial026.stdout:Installing: 2026-02-22T21:35:18.135 INFO:teuthology.orchestra.run.trial026.stdout: ceph x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 6.5 k 2026-02-22T21:35:18.135 INFO:teuthology.orchestra.run.trial026.stdout:Installing dependencies: 2026-02-22T21:35:18.135 INFO:teuthology.orchestra.run.trial026.stdout: abseil-cpp x86_64 20211102.0-4.el9 epel 551 k 2026-02-22T21:35:18.135 INFO:teuthology.orchestra.run.trial026.stdout: ceph-grafana-dashboards noarch 2:19.2.3-689.gf90f4eae.el9 ceph-noarch 31 k 2026-02-22T21:35:18.135 INFO:teuthology.orchestra.run.trial026.stdout: ceph-mds x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 2.4 M 2026-02-22T21:35:18.135 INFO:teuthology.orchestra.run.trial026.stdout: ceph-mgr x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 1.1 M 2026-02-22T21:35:18.136 INFO:teuthology.orchestra.run.trial026.stdout: ceph-mgr-modules-core noarch 2:19.2.3-689.gf90f4eae.el9 ceph-noarch 253 k 2026-02-22T21:35:18.136 INFO:teuthology.orchestra.run.trial026.stdout: ceph-mon x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 4.7 M 2026-02-22T21:35:18.136 INFO:teuthology.orchestra.run.trial026.stdout: ceph-osd x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 17 M 2026-02-22T21:35:18.136 INFO:teuthology.orchestra.run.trial026.stdout: ceph-prometheus-alerts noarch 2:19.2.3-689.gf90f4eae.el9 ceph-noarch 17 k 2026-02-22T21:35:18.136 INFO:teuthology.orchestra.run.trial026.stdout: cephadm noarch 2:19.2.3-689.gf90f4eae.el9 ceph-noarch 769 k 2026-02-22T21:35:18.136 INFO:teuthology.orchestra.run.trial026.stdout: cryptsetup x86_64 2.8.1-3.el9 baseos 351 k 2026-02-22T21:35:18.136 INFO:teuthology.orchestra.run.trial026.stdout: flexiblas x86_64 3.0.4-9.el9 appstream 30 k 2026-02-22T21:35:18.136 INFO:teuthology.orchestra.run.trial026.stdout: flexiblas-netlib x86_64 3.0.4-9.el9 appstream 3.0 M 2026-02-22T21:35:18.136 INFO:teuthology.orchestra.run.trial026.stdout: flexiblas-openblas-openmp x86_64 3.0.4-9.el9 appstream 15 k 2026-02-22T21:35:18.136 INFO:teuthology.orchestra.run.trial026.stdout: grpc-data noarch 1.46.7-10.el9 epel 19 k 2026-02-22T21:35:18.136 INFO:teuthology.orchestra.run.trial026.stdout: ledmon-libs x86_64 1.1.0-3.el9 baseos 40 k 2026-02-22T21:35:18.136 INFO:teuthology.orchestra.run.trial026.stdout: libconfig x86_64 1.7.2-9.el9 baseos 72 k 2026-02-22T21:35:18.136 INFO:teuthology.orchestra.run.trial026.stdout: libgfortran x86_64 11.5.0-14.el9 baseos 794 k 2026-02-22T21:35:18.136 INFO:teuthology.orchestra.run.trial026.stdout: libquadmath x86_64 11.5.0-14.el9 baseos 184 k 2026-02-22T21:35:18.136 INFO:teuthology.orchestra.run.trial026.stdout: libstoragemgmt x86_64 1.10.1-1.el9 appstream 246 k 2026-02-22T21:35:18.136 INFO:teuthology.orchestra.run.trial026.stdout: lua x86_64 5.4.4-4.el9 appstream 188 k 2026-02-22T21:35:18.136 INFO:teuthology.orchestra.run.trial026.stdout: lua-devel x86_64 5.4.4-4.el9 crb 22 k 2026-02-22T21:35:18.137 INFO:teuthology.orchestra.run.trial026.stdout: luarocks noarch 3.9.2-5.el9 epel 151 k 2026-02-22T21:35:18.137 INFO:teuthology.orchestra.run.trial026.stdout: openblas x86_64 0.3.29-1.el9 appstream 42 k 2026-02-22T21:35:18.137 INFO:teuthology.orchestra.run.trial026.stdout: openblas-openmp x86_64 0.3.29-1.el9 appstream 5.3 M 2026-02-22T21:35:18.137 INFO:teuthology.orchestra.run.trial026.stdout: protobuf x86_64 3.14.0-17.el9 appstream 1.0 M 2026-02-22T21:35:18.137 INFO:teuthology.orchestra.run.trial026.stdout: protobuf-compiler x86_64 3.14.0-17.el9 crb 862 k 2026-02-22T21:35:18.137 INFO:teuthology.orchestra.run.trial026.stdout: python3-asyncssh noarch 2.13.2-5.el9 epel 548 k 2026-02-22T21:35:18.137 INFO:teuthology.orchestra.run.trial026.stdout: python3-autocommand noarch 2.2.2-8.el9 epel 29 k 2026-02-22T21:35:18.137 INFO:teuthology.orchestra.run.trial026.stdout: python3-babel noarch 2.9.1-2.el9 appstream 6.0 M 2026-02-22T21:35:18.137 INFO:teuthology.orchestra.run.trial026.stdout: python3-backports-tarfile noarch 1.2.0-1.el9 epel 60 k 2026-02-22T21:35:18.137 INFO:teuthology.orchestra.run.trial026.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2026-02-22T21:35:18.137 INFO:teuthology.orchestra.run.trial026.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2026-02-22T21:35:18.137 INFO:teuthology.orchestra.run.trial026.stdout: python3-certifi noarch 2023.05.07-4.el9 epel 14 k 2026-02-22T21:35:18.137 INFO:teuthology.orchestra.run.trial026.stdout: python3-cffi x86_64 1.14.5-5.el9 baseos 253 k 2026-02-22T21:35:18.137 INFO:teuthology.orchestra.run.trial026.stdout: python3-cheroot noarch 10.0.1-4.el9 epel 173 k 2026-02-22T21:35:18.137 INFO:teuthology.orchestra.run.trial026.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2026-02-22T21:35:18.137 INFO:teuthology.orchestra.run.trial026.stdout: python3-cryptography x86_64 36.0.1-5.el9 baseos 1.2 M 2026-02-22T21:35:18.138 INFO:teuthology.orchestra.run.trial026.stdout: python3-devel x86_64 3.9.25-3.el9 appstream 244 k 2026-02-22T21:35:18.138 INFO:teuthology.orchestra.run.trial026.stdout: python3-google-auth noarch 1:2.45.0-1.el9 epel 254 k 2026-02-22T21:35:18.138 INFO:teuthology.orchestra.run.trial026.stdout: python3-grpcio x86_64 1.46.7-10.el9 epel 2.0 M 2026-02-22T21:35:18.138 INFO:teuthology.orchestra.run.trial026.stdout: python3-grpcio-tools x86_64 1.46.7-10.el9 epel 144 k 2026-02-22T21:35:18.138 INFO:teuthology.orchestra.run.trial026.stdout: python3-isodate noarch 0.6.1-3.el9 epel 56 k 2026-02-22T21:35:18.138 INFO:teuthology.orchestra.run.trial026.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2026-02-22T21:35:18.138 INFO:teuthology.orchestra.run.trial026.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2026-02-22T21:35:18.138 INFO:teuthology.orchestra.run.trial026.stdout: python3-jaraco-collections noarch 3.0.0-8.el9 epel 23 k 2026-02-22T21:35:18.138 INFO:teuthology.orchestra.run.trial026.stdout: python3-jaraco-context noarch 6.0.1-3.el9 epel 20 k 2026-02-22T21:35:18.138 INFO:teuthology.orchestra.run.trial026.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2026-02-22T21:35:18.138 INFO:teuthology.orchestra.run.trial026.stdout: python3-jaraco-text noarch 4.0.0-2.el9 epel 26 k 2026-02-22T21:35:18.138 INFO:teuthology.orchestra.run.trial026.stdout: python3-jinja2 noarch 2.11.3-8.el9 appstream 249 k 2026-02-22T21:35:18.138 INFO:teuthology.orchestra.run.trial026.stdout: python3-kubernetes noarch 1:26.1.0-3.el9 epel 1.0 M 2026-02-22T21:35:18.138 INFO:teuthology.orchestra.run.trial026.stdout: python3-libstoragemgmt x86_64 1.10.1-1.el9 appstream 177 k 2026-02-22T21:35:18.138 INFO:teuthology.orchestra.run.trial026.stdout: python3-logutils noarch 0.3.5-21.el9 epel 46 k 2026-02-22T21:35:18.138 INFO:teuthology.orchestra.run.trial026.stdout: python3-lxml x86_64 4.6.5-3.el9 appstream 1.2 M 2026-02-22T21:35:18.138 INFO:teuthology.orchestra.run.trial026.stdout: python3-mako noarch 1.1.4-6.el9 appstream 172 k 2026-02-22T21:35:18.138 INFO:teuthology.orchestra.run.trial026.stdout: python3-markupsafe x86_64 1.1.1-12.el9 appstream 35 k 2026-02-22T21:35:18.139 INFO:teuthology.orchestra.run.trial026.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2026-02-22T21:35:18.139 INFO:teuthology.orchestra.run.trial026.stdout: python3-msgpack x86_64 1.0.3-2.el9 epel 86 k 2026-02-22T21:35:18.139 INFO:teuthology.orchestra.run.trial026.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2026-02-22T21:35:18.139 INFO:teuthology.orchestra.run.trial026.stdout: python3-numpy x86_64 1:1.23.5-2.el9 appstream 6.1 M 2026-02-22T21:35:18.139 INFO:teuthology.orchestra.run.trial026.stdout: python3-numpy-f2py x86_64 1:1.23.5-2.el9 appstream 442 k 2026-02-22T21:35:18.139 INFO:teuthology.orchestra.run.trial026.stdout: python3-packaging noarch 20.9-5.el9 appstream 77 k 2026-02-22T21:35:18.139 INFO:teuthology.orchestra.run.trial026.stdout: python3-pecan noarch 1.4.2-3.el9 epel 272 k 2026-02-22T21:35:18.139 INFO:teuthology.orchestra.run.trial026.stdout: python3-ply noarch 3.11-14.el9 baseos 106 k 2026-02-22T21:35:18.139 INFO:teuthology.orchestra.run.trial026.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2026-02-22T21:35:18.139 INFO:teuthology.orchestra.run.trial026.stdout: python3-protobuf noarch 3.14.0-17.el9 appstream 267 k 2026-02-22T21:35:18.139 INFO:teuthology.orchestra.run.trial026.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2026-02-22T21:35:18.139 INFO:teuthology.orchestra.run.trial026.stdout: python3-pyasn1 noarch 0.4.8-6.el9 appstream 159 k 2026-02-22T21:35:18.139 INFO:teuthology.orchestra.run.trial026.stdout: python3-pyasn1-modules noarch 0.4.8-6.el9 appstream 279 k 2026-02-22T21:35:18.139 INFO:teuthology.orchestra.run.trial026.stdout: python3-pycparser noarch 2.20-6.el9 baseos 135 k 2026-02-22T21:35:18.139 INFO:teuthology.orchestra.run.trial026.stdout: python3-pyparsing noarch 2.4.7-9.el9 baseos 150 k 2026-02-22T21:35:18.139 INFO:teuthology.orchestra.run.trial026.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2026-02-22T21:35:18.139 INFO:teuthology.orchestra.run.trial026.stdout: python3-requests noarch 2.25.1-10.el9 baseos 126 k 2026-02-22T21:35:18.140 INFO:teuthology.orchestra.run.trial026.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 appstream 54 k 2026-02-22T21:35:18.140 INFO:teuthology.orchestra.run.trial026.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2026-02-22T21:35:18.140 INFO:teuthology.orchestra.run.trial026.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2026-02-22T21:35:18.140 INFO:teuthology.orchestra.run.trial026.stdout: python3-scipy x86_64 1.9.3-2.el9 appstream 19 M 2026-02-22T21:35:18.140 INFO:teuthology.orchestra.run.trial026.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2026-02-22T21:35:18.140 INFO:teuthology.orchestra.run.trial026.stdout: python3-toml noarch 0.10.2-6.el9 appstream 42 k 2026-02-22T21:35:18.140 INFO:teuthology.orchestra.run.trial026.stdout: python3-typing-extensions noarch 4.15.0-1.el9 epel 86 k 2026-02-22T21:35:18.140 INFO:teuthology.orchestra.run.trial026.stdout: python3-urllib3 noarch 1.26.5-7.el9 baseos 218 k 2026-02-22T21:35:18.140 INFO:teuthology.orchestra.run.trial026.stdout: python3-webob noarch 1.8.8-2.el9 epel 230 k 2026-02-22T21:35:18.140 INFO:teuthology.orchestra.run.trial026.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2026-02-22T21:35:18.140 INFO:teuthology.orchestra.run.trial026.stdout: python3-werkzeug noarch 2.0.3-3.el9.1 epel 427 k 2026-02-22T21:35:18.140 INFO:teuthology.orchestra.run.trial026.stdout: python3-xmlsec x86_64 1.3.13-1.el9 epel 48 k 2026-02-22T21:35:18.140 INFO:teuthology.orchestra.run.trial026.stdout: python3-xmltodict noarch 0.12.0-15.el9 epel 22 k 2026-02-22T21:35:18.140 INFO:teuthology.orchestra.run.trial026.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2026-02-22T21:35:18.140 INFO:teuthology.orchestra.run.trial026.stdout: unzip x86_64 6.0-59.el9 baseos 182 k 2026-02-22T21:35:18.140 INFO:teuthology.orchestra.run.trial026.stdout: xmlsec1 x86_64 1.2.29-13.el9 appstream 189 k 2026-02-22T21:35:18.140 INFO:teuthology.orchestra.run.trial026.stdout: xmlsec1-openssl x86_64 1.2.29-13.el9 appstream 90 k 2026-02-22T21:35:18.140 INFO:teuthology.orchestra.run.trial026.stdout: zip x86_64 3.0-35.el9 baseos 266 k 2026-02-22T21:35:18.141 INFO:teuthology.orchestra.run.trial026.stdout:Installing weak dependencies: 2026-02-22T21:35:18.141 INFO:teuthology.orchestra.run.trial026.stdout: ceph-mgr-cephadm noarch 2:19.2.3-689.gf90f4eae.el9 ceph-noarch 150 k 2026-02-22T21:35:18.141 INFO:teuthology.orchestra.run.trial026.stdout: ceph-mgr-dashboard noarch 2:19.2.3-689.gf90f4eae.el9 ceph-noarch 3.8 M 2026-02-22T21:35:18.141 INFO:teuthology.orchestra.run.trial026.stdout: ceph-mgr-diskprediction-local noarch 2:19.2.3-689.gf90f4eae.el9 ceph-noarch 7.4 M 2026-02-22T21:35:18.141 INFO:teuthology.orchestra.run.trial026.stdout: ceph-mgr-k8sevents noarch 2:19.2.3-689.gf90f4eae.el9 ceph-noarch 22 k 2026-02-22T21:35:18.141 INFO:teuthology.orchestra.run.trial026.stdout: ceph-mgr-rook noarch 2:19.2.3-689.gf90f4eae.el9 ceph-noarch 49 k 2026-02-22T21:35:18.141 INFO:teuthology.orchestra.run.trial026.stdout: ceph-volume noarch 2:19.2.3-689.gf90f4eae.el9 ceph-noarch 299 k 2026-02-22T21:35:18.141 INFO:teuthology.orchestra.run.trial026.stdout: python3-influxdb noarch 5.3.1-1.el9 epel 139 k 2026-02-22T21:35:18.141 INFO:teuthology.orchestra.run.trial026.stdout: python3-saml noarch 1.16.0-1.el9 epel 125 k 2026-02-22T21:35:18.141 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:35:18.141 INFO:teuthology.orchestra.run.trial026.stdout:Transaction Summary 2026-02-22T21:35:18.141 INFO:teuthology.orchestra.run.trial026.stdout:====================================================================================== 2026-02-22T21:35:18.141 INFO:teuthology.orchestra.run.trial026.stdout:Install 99 Packages 2026-02-22T21:35:18.141 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:35:18.142 INFO:teuthology.orchestra.run.trial026.stdout:Total download size: 95 M 2026-02-22T21:35:18.142 INFO:teuthology.orchestra.run.trial026.stdout:Installed size: 520 M 2026-02-22T21:35:18.142 INFO:teuthology.orchestra.run.trial026.stdout:Downloading Packages: 2026-02-22T21:35:18.433 INFO:teuthology.orchestra.run.trial004.stdout:Last metadata expiration check: 0:00:43 ago on Sun 22 Feb 2026 09:34:35 PM UTC. 2026-02-22T21:35:18.499 INFO:teuthology.orchestra.run.trial026.stdout:(1/99): ceph-19.2.3-689.gf90f4eae.el9.x86_64.rp 123 kB/s | 6.5 kB 00:00 2026-02-22T21:35:18.517 INFO:teuthology.orchestra.run.trial004.stdout:Dependencies resolved. 2026-02-22T21:35:18.517 INFO:teuthology.orchestra.run.trial004.stdout:================================================================================ 2026-02-22T21:35:18.517 INFO:teuthology.orchestra.run.trial004.stdout: Package Arch Version Repo Size 2026-02-22T21:35:18.517 INFO:teuthology.orchestra.run.trial004.stdout:================================================================================ 2026-02-22T21:35:18.517 INFO:teuthology.orchestra.run.trial004.stdout:Installing: 2026-02-22T21:35:18.517 INFO:teuthology.orchestra.run.trial004.stdout: rbd-mirror x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 3.1 M 2026-02-22T21:35:18.517 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:35:18.517 INFO:teuthology.orchestra.run.trial004.stdout:Transaction Summary 2026-02-22T21:35:18.517 INFO:teuthology.orchestra.run.trial004.stdout:================================================================================ 2026-02-22T21:35:18.517 INFO:teuthology.orchestra.run.trial004.stdout:Install 1 Package 2026-02-22T21:35:18.517 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:35:18.518 INFO:teuthology.orchestra.run.trial004.stdout:Total download size: 3.1 M 2026-02-22T21:35:18.518 INFO:teuthology.orchestra.run.trial004.stdout:Installed size: 13 M 2026-02-22T21:35:18.518 INFO:teuthology.orchestra.run.trial004.stdout:Downloading Packages: 2026-02-22T21:35:18.616 INFO:teuthology.orchestra.run.trial026.stdout:(2/99): ceph-mgr-19.2.3-689.gf90f4eae.el9.x86_6 6.3 MB/s | 1.1 MB 00:00 2026-02-22T21:35:18.746 INFO:teuthology.orchestra.run.trial026.stdout:(3/99): ceph-mds-19.2.3-689.gf90f4eae.el9.x86_6 8.1 MB/s | 2.4 MB 00:00 2026-02-22T21:35:18.783 INFO:teuthology.orchestra.run.trial026.stdout:(4/99): ceph-grafana-dashboards-19.2.3-689.gf90 857 kB/s | 31 kB 00:00 2026-02-22T21:35:18.822 INFO:teuthology.orchestra.run.trial026.stdout:(5/99): ceph-mgr-cephadm-19.2.3-689.gf90f4eae.e 3.7 MB/s | 150 kB 00:00 2026-02-22T21:35:19.244 INFO:teuthology.orchestra.run.trial026.stdout:(6/99): ceph-mon-19.2.3-689.gf90f4eae.el9.x86_6 6.4 MB/s | 4.7 MB 00:00 2026-02-22T21:35:19.254 INFO:teuthology.orchestra.run.trial026.stdout:(7/99): ceph-mgr-dashboard-19.2.3-689.gf90f4eae 8.8 MB/s | 3.8 MB 00:00 2026-02-22T21:35:19.298 INFO:teuthology.orchestra.run.trial026.stdout:(8/99): ceph-mgr-k8sevents-19.2.3-689.gf90f4eae 495 kB/s | 22 kB 00:00 2026-02-22T21:35:19.359 INFO:teuthology.orchestra.run.trial026.stdout:(9/99): ceph-mgr-modules-core-19.2.3-689.gf90f4 4.1 MB/s | 253 kB 00:00 2026-02-22T21:35:19.420 INFO:teuthology.orchestra.run.trial026.stdout:(10/99): ceph-mgr-rook-19.2.3-689.gf90f4eae.el9 811 kB/s | 49 kB 00:00 2026-02-22T21:35:19.486 INFO:teuthology.orchestra.run.trial026.stdout:(11/99): ceph-prometheus-alerts-19.2.3-689.gf90 254 kB/s | 17 kB 00:00 2026-02-22T21:35:19.909 INFO:teuthology.orchestra.run.trial026.stdout:(12/99): ceph-osd-19.2.3-689.gf90f4eae.el9.x86_ 13 MB/s | 17 MB 00:01 2026-02-22T21:35:19.989 INFO:teuthology.orchestra.run.trial026.stdout:(13/99): cephadm-19.2.3-689.gf90f4eae.el9.noarc 9.5 MB/s | 769 kB 00:00 2026-02-22T21:35:20.019 INFO:teuthology.orchestra.run.trial026.stdout:(14/99): ceph-volume-19.2.3-689.gf90f4eae.el9.n 560 kB/s | 299 kB 00:00 2026-02-22T21:35:20.042 INFO:teuthology.orchestra.run.trial026.stdout:(15/99): ceph-mgr-diskprediction-local-19.2.3-6 9.3 MB/s | 7.4 MB 00:00 2026-02-22T21:35:20.046 INFO:teuthology.orchestra.run.trial004.stdout:rbd-mirror-19.2.3-689.gf90f4eae.el9.x86_64.rpm 2.0 MB/s | 3.1 MB 00:01 2026-02-22T21:35:20.046 INFO:teuthology.orchestra.run.trial004.stdout:-------------------------------------------------------------------------------- 2026-02-22T21:35:20.047 INFO:teuthology.orchestra.run.trial004.stdout:Total 2.0 MB/s | 3.1 MB 00:01 2026-02-22T21:35:20.047 INFO:teuthology.orchestra.run.trial004.stdout:Running transaction check 2026-02-22T21:35:20.051 INFO:teuthology.orchestra.run.trial004.stdout:Transaction check succeeded. 2026-02-22T21:35:20.051 INFO:teuthology.orchestra.run.trial004.stdout:Running transaction test 2026-02-22T21:35:20.091 INFO:teuthology.orchestra.run.trial004.stdout:Transaction test succeeded. 2026-02-22T21:35:20.091 INFO:teuthology.orchestra.run.trial004.stdout:Running transaction 2026-02-22T21:35:20.291 INFO:teuthology.orchestra.run.trial004.stdout: Preparing : 1/1 2026-02-22T21:35:20.292 INFO:teuthology.orchestra.run.trial004.stdout: Installing : rbd-mirror-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T21:35:20.315 INFO:teuthology.orchestra.run.trial004.stdout: Running scriptlet: rbd-mirror-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T21:35:20.315 INFO:teuthology.orchestra.run.trial004.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-22T21:35:20.315 INFO:teuthology.orchestra.run.trial004.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2026-02-22T21:35:20.315 INFO:teuthology.orchestra.run.trial004.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2026-02-22T21:35:20.315 INFO:teuthology.orchestra.run.trial004.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2026-02-22T21:35:20.315 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:35:20.459 INFO:teuthology.orchestra.run.trial026.stdout:(16/99): ledmon-libs-1.1.0-3.el9.x86_64.rpm 92 kB/s | 40 kB 00:00 2026-02-22T21:35:20.595 INFO:teuthology.orchestra.run.trial026.stdout:(17/99): libconfig-1.7.2-9.el9.x86_64.rpm 130 kB/s | 72 kB 00:00 2026-02-22T21:35:20.648 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : rbd-mirror-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T21:35:20.648 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:35:20.648 INFO:teuthology.orchestra.run.trial004.stdout:Installed: 2026-02-22T21:35:20.648 INFO:teuthology.orchestra.run.trial004.stdout: rbd-mirror-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:35:20.648 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:35:20.648 INFO:teuthology.orchestra.run.trial004.stdout:Complete! 2026-02-22T21:35:20.676 DEBUG:teuthology.orchestra.run.trial004:> sudo yum -y install rbd-nbd 2026-02-22T21:35:20.872 INFO:teuthology.orchestra.run.trial026.stdout:(18/99): cryptsetup-2.8.1-3.el9.x86_64.rpm 397 kB/s | 351 kB 00:00 2026-02-22T21:35:20.910 INFO:teuthology.orchestra.run.trial026.stdout:(19/99): libquadmath-11.5.0-14.el9.x86_64.rpm 587 kB/s | 184 kB 00:00 2026-02-22T21:35:20.988 INFO:teuthology.orchestra.run.trial004.stdout:Last metadata expiration check: 0:00:45 ago on Sun 22 Feb 2026 09:34:35 PM UTC. 2026-02-22T21:35:21.071 INFO:teuthology.orchestra.run.trial004.stdout:Dependencies resolved. 2026-02-22T21:35:21.071 INFO:teuthology.orchestra.run.trial004.stdout:================================================================================ 2026-02-22T21:35:21.071 INFO:teuthology.orchestra.run.trial004.stdout: Package Architecture Version Repository Size 2026-02-22T21:35:21.071 INFO:teuthology.orchestra.run.trial004.stdout:================================================================================ 2026-02-22T21:35:21.071 INFO:teuthology.orchestra.run.trial004.stdout:Installing: 2026-02-22T21:35:21.071 INFO:teuthology.orchestra.run.trial004.stdout: rbd-nbd x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 170 k 2026-02-22T21:35:21.072 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:35:21.072 INFO:teuthology.orchestra.run.trial004.stdout:Transaction Summary 2026-02-22T21:35:21.072 INFO:teuthology.orchestra.run.trial004.stdout:================================================================================ 2026-02-22T21:35:21.072 INFO:teuthology.orchestra.run.trial004.stdout:Install 1 Package 2026-02-22T21:35:21.072 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:35:21.072 INFO:teuthology.orchestra.run.trial004.stdout:Total download size: 170 k 2026-02-22T21:35:21.072 INFO:teuthology.orchestra.run.trial004.stdout:Installed size: 490 k 2026-02-22T21:35:21.072 INFO:teuthology.orchestra.run.trial004.stdout:Downloading Packages: 2026-02-22T21:35:21.159 INFO:teuthology.orchestra.run.trial004.stdout:rbd-nbd-19.2.3-689.gf90f4eae.el9.x86_64.rpm 1.9 MB/s | 170 kB 00:00 2026-02-22T21:35:21.159 INFO:teuthology.orchestra.run.trial004.stdout:-------------------------------------------------------------------------------- 2026-02-22T21:35:21.159 INFO:teuthology.orchestra.run.trial004.stdout:Total 1.9 MB/s | 170 kB 00:00 2026-02-22T21:35:21.159 INFO:teuthology.orchestra.run.trial004.stdout:Running transaction check 2026-02-22T21:35:21.163 INFO:teuthology.orchestra.run.trial004.stdout:Transaction check succeeded. 2026-02-22T21:35:21.163 INFO:teuthology.orchestra.run.trial004.stdout:Running transaction test 2026-02-22T21:35:21.192 INFO:teuthology.orchestra.run.trial026.stdout:(20/99): python3-cffi-1.14.5-5.el9.x86_64.rpm 793 kB/s | 253 kB 00:00 2026-02-22T21:35:21.194 INFO:teuthology.orchestra.run.trial004.stdout:Transaction test succeeded. 2026-02-22T21:35:21.194 INFO:teuthology.orchestra.run.trial004.stdout:Running transaction 2026-02-22T21:35:21.249 INFO:teuthology.orchestra.run.trial004.stdout: Preparing : 1/1 2026-02-22T21:35:21.266 INFO:teuthology.orchestra.run.trial004.stdout: Installing : rbd-nbd-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T21:35:21.350 INFO:teuthology.orchestra.run.trial004.stdout: Running scriptlet: rbd-nbd-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T21:35:21.354 INFO:teuthology.orchestra.run.trial026.stdout:(21/99): python3-ply-3.11-14.el9.noarch.rpm 656 kB/s | 106 kB 00:00 2026-02-22T21:35:21.407 INFO:teuthology.orchestra.run.trial004.stdout: Verifying : rbd-nbd-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T21:35:21.408 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:35:21.408 INFO:teuthology.orchestra.run.trial004.stdout:Installed: 2026-02-22T21:35:21.408 INFO:teuthology.orchestra.run.trial004.stdout: rbd-nbd-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:35:21.408 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:35:21.408 INFO:teuthology.orchestra.run.trial004.stdout:Complete! 2026-02-22T21:35:21.436 DEBUG:teuthology.orchestra.run.trial004:> sudo yum -y install rbd-mirror 2026-02-22T21:35:21.516 INFO:teuthology.orchestra.run.trial026.stdout:(22/99): python3-pycparser-2.20-6.el9.noarch.rp 832 kB/s | 135 kB 00:00 2026-02-22T21:35:21.747 INFO:teuthology.orchestra.run.trial004.stdout:Last metadata expiration check: 0:00:46 ago on Sun 22 Feb 2026 09:34:35 PM UTC. 2026-02-22T21:35:21.757 INFO:teuthology.orchestra.run.trial026.stdout:(23/99): python3-pyparsing-2.4.7-9.el9.noarch.r 625 kB/s | 150 kB 00:00 2026-02-22T21:35:21.813 INFO:teuthology.orchestra.run.trial004.stdout:Package rbd-mirror-2:19.2.3-689.gf90f4eae.el9.x86_64 is already installed. 2026-02-22T21:35:21.831 INFO:teuthology.orchestra.run.trial004.stdout:Dependencies resolved. 2026-02-22T21:35:21.832 INFO:teuthology.orchestra.run.trial004.stdout:Nothing to do. 2026-02-22T21:35:21.832 INFO:teuthology.orchestra.run.trial004.stdout:Complete! 2026-02-22T21:35:21.854 DEBUG:teuthology.parallel:result is None 2026-02-22T21:35:21.921 INFO:teuthology.orchestra.run.trial026.stdout:(24/99): python3-requests-2.25.1-10.el9.noarch. 771 kB/s | 126 kB 00:00 2026-02-22T21:35:22.240 INFO:teuthology.orchestra.run.trial026.stdout:(25/99): python3-urllib3-1.26.5-7.el9.noarch.rp 682 kB/s | 218 kB 00:00 2026-02-22T21:35:22.388 INFO:teuthology.orchestra.run.trial026.stdout:(26/99): python3-cryptography-36.0.1-5.el9.x86_ 862 kB/s | 1.2 MB 00:01 2026-02-22T21:35:22.412 INFO:teuthology.orchestra.run.trial026.stdout:(27/99): libgfortran-11.5.0-14.el9.x86_64.rpm 407 kB/s | 794 kB 00:01 2026-02-22T21:35:22.455 INFO:teuthology.orchestra.run.trial026.stdout:(28/99): flexiblas-3.0.4-9.el9.x86_64.rpm 690 kB/s | 30 kB 00:00 2026-02-22T21:35:22.481 INFO:teuthology.orchestra.run.trial026.stdout:(29/99): unzip-6.0-59.el9.x86_64.rpm 756 kB/s | 182 kB 00:00 2026-02-22T21:35:22.523 INFO:teuthology.orchestra.run.trial026.stdout:(30/99): flexiblas-openblas-openmp-3.0.4-9.el9. 354 kB/s | 15 kB 00:00 2026-02-22T21:35:22.585 INFO:teuthology.orchestra.run.trial026.stdout:(31/99): flexiblas-netlib-3.0.4-9.el9.x86_64.rp 23 MB/s | 3.0 MB 00:00 2026-02-22T21:35:22.587 INFO:teuthology.orchestra.run.trial026.stdout:(32/99): libstoragemgmt-1.10.1-1.el9.x86_64.rpm 3.8 MB/s | 246 kB 00:00 2026-02-22T21:35:22.608 INFO:teuthology.orchestra.run.trial026.stdout:(33/99): lua-5.4.4-4.el9.x86_64.rpm 7.8 MB/s | 188 kB 00:00 2026-02-22T21:35:22.609 INFO:teuthology.orchestra.run.trial026.stdout:(34/99): openblas-0.3.29-1.el9.x86_64.rpm 1.9 MB/s | 42 kB 00:00 2026-02-22T21:35:22.663 INFO:teuthology.orchestra.run.trial026.stdout:(35/99): openblas-openmp-0.3.29-1.el9.x86_64.rp 97 MB/s | 5.3 MB 00:00 2026-02-22T21:35:22.691 INFO:teuthology.orchestra.run.trial026.stdout:(36/99): protobuf-3.14.0-17.el9.x86_64.rpm 12 MB/s | 1.0 MB 00:00 2026-02-22T21:35:22.704 INFO:teuthology.orchestra.run.trial026.stdout:(37/99): zip-3.0-35.el9.x86_64.rpm 844 kB/s | 266 kB 00:00 2026-02-22T21:35:22.720 INFO:teuthology.orchestra.run.trial026.stdout:(38/99): python3-babel-2.9.1-2.el9.noarch.rpm 105 MB/s | 6.0 MB 00:00 2026-02-22T21:35:22.721 INFO:teuthology.orchestra.run.trial026.stdout:(39/99): python3-devel-3.9.25-3.el9.x86_64.rpm 7.9 MB/s | 244 kB 00:00 2026-02-22T21:35:22.745 INFO:teuthology.orchestra.run.trial026.stdout:(40/99): python3-libstoragemgmt-1.10.1-1.el9.x8 7.1 MB/s | 177 kB 00:00 2026-02-22T21:35:22.767 INFO:teuthology.orchestra.run.trial026.stdout:(41/99): python3-lxml-4.6.5-3.el9.x86_64.rpm 26 MB/s | 1.2 MB 00:00 2026-02-22T21:35:22.768 INFO:teuthology.orchestra.run.trial026.stdout:(42/99): python3-mako-1.1.4-6.el9.noarch.rpm 7.0 MB/s | 172 kB 00:00 2026-02-22T21:35:22.790 INFO:teuthology.orchestra.run.trial026.stdout:(43/99): python3-jinja2-2.11.3-8.el9.noarch.rpm 2.8 MB/s | 249 kB 00:00 2026-02-22T21:35:22.790 INFO:teuthology.orchestra.run.trial026.stdout:(44/99): python3-markupsafe-1.1.1-12.el9.x86_64 1.5 MB/s | 35 kB 00:00 2026-02-22T21:35:22.813 INFO:teuthology.orchestra.run.trial026.stdout:(45/99): python3-packaging-20.9-5.el9.noarch.rp 3.3 MB/s | 77 kB 00:00 2026-02-22T21:35:22.826 INFO:teuthology.orchestra.run.trial026.stdout:(46/99): python3-numpy-1.23.5-2.el9.x86_64.rpm 108 MB/s | 6.1 MB 00:00 2026-02-22T21:35:22.834 INFO:teuthology.orchestra.run.trial026.stdout:(47/99): python3-numpy-f2py-1.23.5-2.el9.x86_64 9.7 MB/s | 442 kB 00:00 2026-02-22T21:35:22.837 INFO:teuthology.orchestra.run.trial026.stdout:(48/99): python3-protobuf-3.14.0-17.el9.noarch. 11 MB/s | 267 kB 00:00 2026-02-22T21:35:22.850 INFO:teuthology.orchestra.run.trial026.stdout:(49/99): python3-pyasn1-0.4.8-6.el9.noarch.rpm 6.4 MB/s | 159 kB 00:00 2026-02-22T21:35:22.856 INFO:teuthology.orchestra.run.trial026.stdout:(50/99): python3-pyasn1-modules-0.4.8-6.el9.noa 12 MB/s | 279 kB 00:00 2026-02-22T21:35:22.860 INFO:teuthology.orchestra.run.trial026.stdout:(51/99): python3-requests-oauthlib-1.3.0-12.el9 2.3 MB/s | 54 kB 00:00 2026-02-22T21:35:22.877 INFO:teuthology.orchestra.run.trial026.stdout:(52/99): python3-toml-0.10.2-6.el9.noarch.rpm 2.0 MB/s | 42 kB 00:00 2026-02-22T21:35:22.884 INFO:teuthology.orchestra.run.trial026.stdout:(53/99): xmlsec1-1.2.29-13.el9.x86_64.rpm 7.5 MB/s | 189 kB 00:00 2026-02-22T21:35:22.898 INFO:teuthology.orchestra.run.trial026.stdout:(54/99): xmlsec1-openssl-1.2.29-13.el9.x86_64.r 4.2 MB/s | 90 kB 00:00 2026-02-22T21:35:22.967 INFO:teuthology.orchestra.run.trial026.stdout:(55/99): lua-devel-5.4.4-4.el9.x86_64.rpm 269 kB/s | 22 kB 00:00 2026-02-22T21:35:23.007 INFO:teuthology.orchestra.run.trial026.stdout:(56/99): python3-scipy-1.9.3-2.el9.x86_64.rpm 122 MB/s | 19 MB 00:00 2026-02-22T21:35:23.112 INFO:teuthology.orchestra.run.trial026.stdout:(57/99): grpc-data-1.46.7-10.el9.noarch.rpm 192 kB/s | 19 kB 00:00 2026-02-22T21:35:23.167 INFO:teuthology.orchestra.run.trial026.stdout:(58/99): abseil-cpp-20211102.0-4.el9.x86_64.rpm 2.7 MB/s | 551 kB 00:00 2026-02-22T21:35:23.169 INFO:teuthology.orchestra.run.trial026.stdout:(59/99): luarocks-3.9.2-5.el9.noarch.rpm 2.6 MB/s | 151 kB 00:00 2026-02-22T21:35:23.189 INFO:teuthology.orchestra.run.trial026.stdout:(60/99): python3-autocommand-2.2.2-8.el9.noarch 1.5 MB/s | 29 kB 00:00 2026-02-22T21:35:23.207 INFO:teuthology.orchestra.run.trial026.stdout:(61/99): python3-asyncssh-2.13.2-5.el9.noarch.r 14 MB/s | 548 kB 00:00 2026-02-22T21:35:23.209 INFO:teuthology.orchestra.run.trial026.stdout:(62/99): python3-backports-tarfile-1.2.0-1.el9. 3.0 MB/s | 60 kB 00:00 2026-02-22T21:35:23.227 INFO:teuthology.orchestra.run.trial026.stdout:(63/99): python3-bcrypt-3.2.2-1.el9.x86_64.rpm 2.1 MB/s | 43 kB 00:00 2026-02-22T21:35:23.229 INFO:teuthology.orchestra.run.trial026.stdout:(64/99): python3-cachetools-4.2.4-1.el9.noarch. 1.6 MB/s | 32 kB 00:00 2026-02-22T21:35:23.247 INFO:teuthology.orchestra.run.trial026.stdout:(65/99): python3-certifi-2023.05.07-4.el9.noarc 720 kB/s | 14 kB 00:00 2026-02-22T21:35:23.249 INFO:teuthology.orchestra.run.trial026.stdout:(66/99): python3-cheroot-10.0.1-4.el9.noarch.rp 8.4 MB/s | 173 kB 00:00 2026-02-22T21:35:23.268 INFO:teuthology.orchestra.run.trial026.stdout:(67/99): python3-cherrypy-18.6.1-2.el9.noarch.r 16 MB/s | 358 kB 00:00 2026-02-22T21:35:23.270 INFO:teuthology.orchestra.run.trial026.stdout:(68/99): python3-google-auth-2.45.0-1.el9.noarc 12 MB/s | 254 kB 00:00 2026-02-22T21:35:23.290 INFO:teuthology.orchestra.run.trial026.stdout:(69/99): python3-grpcio-tools-1.46.7-10.el9.x86 7.0 MB/s | 144 kB 00:00 2026-02-22T21:35:23.312 INFO:teuthology.orchestra.run.trial026.stdout:(70/99): python3-grpcio-1.46.7-10.el9.x86_64.rp 46 MB/s | 2.0 MB 00:00 2026-02-22T21:35:23.312 INFO:teuthology.orchestra.run.trial026.stdout:(71/99): python3-influxdb-5.3.1-1.el9.noarch.rp 6.1 MB/s | 139 kB 00:00 2026-02-22T21:35:23.332 INFO:teuthology.orchestra.run.trial026.stdout:(72/99): python3-isodate-0.6.1-3.el9.noarch.rpm 2.7 MB/s | 56 kB 00:00 2026-02-22T21:35:23.332 INFO:teuthology.orchestra.run.trial026.stdout:(73/99): python3-jaraco-8.2.1-3.el9.noarch.rpm 548 kB/s | 11 kB 00:00 2026-02-22T21:35:23.351 INFO:teuthology.orchestra.run.trial026.stdout:(74/99): python3-jaraco-classes-3.2.1-5.el9.noa 899 kB/s | 18 kB 00:00 2026-02-22T21:35:23.352 INFO:teuthology.orchestra.run.trial026.stdout:(75/99): python3-jaraco-collections-3.0.0-8.el9 1.1 MB/s | 23 kB 00:00 2026-02-22T21:35:23.371 INFO:teuthology.orchestra.run.trial026.stdout:(76/99): python3-jaraco-context-6.0.1-3.el9.noa 992 kB/s | 20 kB 00:00 2026-02-22T21:35:23.372 INFO:teuthology.orchestra.run.trial026.stdout:(77/99): python3-jaraco-functools-3.5.0-2.el9.n 977 kB/s | 19 kB 00:00 2026-02-22T21:35:23.391 INFO:teuthology.orchestra.run.trial026.stdout:(78/99): python3-jaraco-text-4.0.0-2.el9.noarch 1.3 MB/s | 26 kB 00:00 2026-02-22T21:35:23.412 INFO:teuthology.orchestra.run.trial026.stdout:(79/99): python3-logutils-0.3.5-21.el9.noarch.r 2.3 MB/s | 46 kB 00:00 2026-02-22T21:35:23.414 INFO:teuthology.orchestra.run.trial026.stdout:(80/99): python3-kubernetes-26.1.0-3.el9.noarch 24 MB/s | 1.0 MB 00:00 2026-02-22T21:35:23.432 INFO:teuthology.orchestra.run.trial026.stdout:(81/99): python3-more-itertools-8.12.0-2.el9.no 3.8 MB/s | 79 kB 00:00 2026-02-22T21:35:23.434 INFO:teuthology.orchestra.run.trial026.stdout:(82/99): python3-msgpack-1.0.3-2.el9.x86_64.rpm 4.2 MB/s | 86 kB 00:00 2026-02-22T21:35:23.452 INFO:teuthology.orchestra.run.trial026.stdout:(83/99): python3-natsort-7.1.1-5.el9.noarch.rpm 2.8 MB/s | 58 kB 00:00 2026-02-22T21:35:23.455 INFO:teuthology.orchestra.run.trial026.stdout:(84/99): python3-pecan-1.4.2-3.el9.noarch.rpm 13 MB/s | 272 kB 00:00 2026-02-22T21:35:23.472 INFO:teuthology.orchestra.run.trial026.stdout:(85/99): python3-portend-3.1.0-2.el9.noarch.rpm 833 kB/s | 16 kB 00:00 2026-02-22T21:35:23.475 INFO:teuthology.orchestra.run.trial026.stdout:(86/99): python3-pyOpenSSL-21.0.0-1.el9.noarch. 4.4 MB/s | 90 kB 00:00 2026-02-22T21:35:23.491 INFO:teuthology.orchestra.run.trial026.stdout:(87/99): python3-repoze-lru-0.7-16.el9.noarch.r 1.5 MB/s | 31 kB 00:00 2026-02-22T21:35:23.495 INFO:teuthology.orchestra.run.trial026.stdout:(88/99): python3-routes-2.5.1-5.el9.noarch.rpm 9.0 MB/s | 188 kB 00:00 2026-02-22T21:35:23.512 INFO:teuthology.orchestra.run.trial026.stdout:(89/99): python3-rsa-4.9-2.el9.noarch.rpm 2.9 MB/s | 59 kB 00:00 2026-02-22T21:35:23.516 INFO:teuthology.orchestra.run.trial026.stdout:(90/99): python3-saml-1.16.0-1.el9.noarch.rpm 6.0 MB/s | 125 kB 00:00 2026-02-22T21:35:23.532 INFO:teuthology.orchestra.run.trial026.stdout:(91/99): python3-tempora-5.0.0-2.el9.noarch.rpm 1.8 MB/s | 36 kB 00:00 2026-02-22T21:35:23.536 INFO:teuthology.orchestra.run.trial026.stdout:(92/99): python3-typing-extensions-4.15.0-1.el9 4.2 MB/s | 86 kB 00:00 2026-02-22T21:35:23.553 INFO:teuthology.orchestra.run.trial026.stdout:(93/99): python3-webob-1.8.8-2.el9.noarch.rpm 11 MB/s | 230 kB 00:00 2026-02-22T21:35:23.556 INFO:teuthology.orchestra.run.trial026.stdout:(94/99): python3-websocket-client-1.2.3-2.el9.n 4.4 MB/s | 90 kB 00:00 2026-02-22T21:35:23.574 INFO:teuthology.orchestra.run.trial026.stdout:(95/99): python3-werkzeug-2.0.3-3.el9.1.noarch. 19 MB/s | 427 kB 00:00 2026-02-22T21:35:23.576 INFO:teuthology.orchestra.run.trial026.stdout:(96/99): python3-xmlsec-1.3.13-1.el9.x86_64.rpm 2.4 MB/s | 48 kB 00:00 2026-02-22T21:35:23.594 INFO:teuthology.orchestra.run.trial026.stdout:(97/99): python3-xmltodict-0.12.0-15.el9.noarch 1.1 MB/s | 22 kB 00:00 2026-02-22T21:35:23.596 INFO:teuthology.orchestra.run.trial026.stdout:(98/99): python3-zc-lockfile-2.0-10.el9.noarch. 990 kB/s | 20 kB 00:00 2026-02-22T21:35:24.074 INFO:teuthology.orchestra.run.trial026.stdout:(99/99): protobuf-compiler-3.14.0-17.el9.x86_64 733 kB/s | 862 kB 00:01 2026-02-22T21:35:24.080 INFO:teuthology.orchestra.run.trial026.stdout:-------------------------------------------------------------------------------- 2026-02-22T21:35:24.080 INFO:teuthology.orchestra.run.trial026.stdout:Total 16 MB/s | 95 MB 00:05 2026-02-22T21:35:24.506 INFO:teuthology.orchestra.run.trial026.stdout:Running transaction check 2026-02-22T21:35:24.545 INFO:teuthology.orchestra.run.trial026.stdout:Transaction check succeeded. 2026-02-22T21:35:24.546 INFO:teuthology.orchestra.run.trial026.stdout:Running transaction test 2026-02-22T21:35:24.967 INFO:teuthology.orchestra.run.trial026.stdout:Transaction test succeeded. 2026-02-22T21:35:24.967 INFO:teuthology.orchestra.run.trial026.stdout:Running transaction 2026-02-22T21:35:25.431 INFO:teuthology.orchestra.run.trial026.stdout: Preparing : 1/1 2026-02-22T21:35:25.433 INFO:teuthology.orchestra.run.trial026.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 1/99 2026-02-22T21:35:25.464 INFO:teuthology.orchestra.run.trial026.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 2/99 2026-02-22T21:35:25.469 INFO:teuthology.orchestra.run.trial026.stdout: Installing : python3-werkzeug-2.0.3-3.el9.1.noarch 3/99 2026-02-22T21:35:25.491 INFO:teuthology.orchestra.run.trial026.stdout: Installing : xmlsec1-1.2.29-13.el9.x86_64 4/99 2026-02-22T21:35:25.497 INFO:teuthology.orchestra.run.trial026.stdout: Installing : python3-pyasn1-0.4.8-6.el9.noarch 5/99 2026-02-22T21:35:25.531 INFO:teuthology.orchestra.run.trial026.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 6/99 2026-02-22T21:35:25.540 INFO:teuthology.orchestra.run.trial026.stdout: Installing : python3-lxml-4.6.5-3.el9.x86_64 7/99 2026-02-22T21:35:25.545 INFO:teuthology.orchestra.run.trial026.stdout: Installing : protobuf-3.14.0-17.el9.x86_64 8/99 2026-02-22T21:35:25.548 INFO:teuthology.orchestra.run.trial026.stdout: Installing : lua-5.4.4-4.el9.x86_64 9/99 2026-02-22T21:35:25.552 INFO:teuthology.orchestra.run.trial026.stdout: Installing : flexiblas-3.0.4-9.el9.x86_64 10/99 2026-02-22T21:35:25.575 INFO:teuthology.orchestra.run.trial026.stdout: Installing : unzip-6.0-59.el9.x86_64 11/99 2026-02-22T21:35:25.588 INFO:teuthology.orchestra.run.trial026.stdout: Installing : python3-urllib3-1.26.5-7.el9.noarch 12/99 2026-02-22T21:35:25.591 INFO:teuthology.orchestra.run.trial026.stdout: Installing : python3-requests-2.25.1-10.el9.noarch 13/99 2026-02-22T21:35:25.597 INFO:teuthology.orchestra.run.trial026.stdout: Installing : libquadmath-11.5.0-14.el9.x86_64 14/99 2026-02-22T21:35:25.599 INFO:teuthology.orchestra.run.trial026.stdout: Installing : libgfortran-11.5.0-14.el9.x86_64 15/99 2026-02-22T21:35:25.610 INFO:teuthology.orchestra.run.trial026.stdout: Installing : ledmon-libs-1.1.0-3.el9.x86_64 16/99 2026-02-22T21:35:25.616 INFO:teuthology.orchestra.run.trial026.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 17/99 2026-02-22T21:35:25.641 INFO:teuthology.orchestra.run.trial026.stdout: Installing : zip-3.0-35.el9.x86_64 18/99 2026-02-22T21:35:25.644 INFO:teuthology.orchestra.run.trial026.stdout: Installing : luarocks-3.9.2-5.el9.noarch 19/99 2026-02-22T21:35:25.651 INFO:teuthology.orchestra.run.trial026.stdout: Installing : lua-devel-5.4.4-4.el9.x86_64 20/99 2026-02-22T21:35:25.675 INFO:teuthology.orchestra.run.trial026.stdout: Installing : protobuf-compiler-3.14.0-17.el9.x86_64 21/99 2026-02-22T21:35:25.727 INFO:teuthology.orchestra.run.trial026.stdout: Installing : python3-mako-1.1.4-6.el9.noarch 22/99 2026-02-22T21:35:25.740 INFO:teuthology.orchestra.run.trial026.stdout: Installing : python3-pyasn1-modules-0.4.8-6.el9.noarch 23/99 2026-02-22T21:35:25.742 INFO:teuthology.orchestra.run.trial026.stdout: Installing : python3-rsa-4.9-2.el9.noarch 24/99 2026-02-22T21:35:25.749 INFO:teuthology.orchestra.run.trial026.stdout: Installing : xmlsec1-openssl-1.2.29-13.el9.x86_64 25/99 2026-02-22T21:35:25.755 INFO:teuthology.orchestra.run.trial026.stdout: Installing : python3-xmlsec-1.3.13-1.el9.x86_64 26/99 2026-02-22T21:35:25.759 INFO:teuthology.orchestra.run.trial026.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 27/99 2026-02-22T21:35:25.762 INFO:teuthology.orchestra.run.trial026.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 28/99 2026-02-22T21:35:25.777 INFO:teuthology.orchestra.run.trial026.stdout: Installing : python3-xmltodict-0.12.0-15.el9.noarch 29/99 2026-02-22T21:35:25.798 INFO:teuthology.orchestra.run.trial026.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 30/99 2026-02-22T21:35:25.802 INFO:teuthology.orchestra.run.trial026.stdout: Installing : python3-webob-1.8.8-2.el9.noarch 31/99 2026-02-22T21:35:25.807 INFO:teuthology.orchestra.run.trial026.stdout: Installing : python3-typing-extensions-4.15.0-1.el9.noarch 32/99 2026-02-22T21:35:25.818 INFO:teuthology.orchestra.run.trial026.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 33/99 2026-02-22T21:35:25.827 INFO:teuthology.orchestra.run.trial026.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 34/99 2026-02-22T21:35:25.833 INFO:teuthology.orchestra.run.trial026.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 35/99 2026-02-22T21:35:25.853 INFO:teuthology.orchestra.run.trial026.stdout: Installing : python3-msgpack-1.0.3-2.el9.x86_64 36/99 2026-02-22T21:35:25.862 INFO:teuthology.orchestra.run.trial026.stdout: Installing : python3-influxdb-5.3.1-1.el9.noarch 37/99 2026-02-22T21:35:25.917 INFO:teuthology.orchestra.run.trial026.stdout: Installing : python3-logutils-0.3.5-21.el9.noarch 38/99 2026-02-22T21:35:25.929 INFO:teuthology.orchestra.run.trial026.stdout: Installing : python3-pecan-1.4.2-3.el9.noarch 39/99 2026-02-22T21:35:25.944 INFO:teuthology.orchestra.run.trial026.stdout: Installing : python3-isodate-0.6.1-3.el9.noarch 40/99 2026-02-22T21:35:25.949 INFO:teuthology.orchestra.run.trial026.stdout: Installing : python3-saml-1.16.0-1.el9.noarch 41/99 2026-02-22T21:35:25.957 INFO:teuthology.orchestra.run.trial026.stdout: Installing : python3-certifi-2023.05.07-4.el9.noarch 42/99 2026-02-22T21:35:25.997 INFO:teuthology.orchestra.run.trial026.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 43/99 2026-02-22T21:35:26.325 INFO:teuthology.orchestra.run.trial026.stdout: Installing : python3-google-auth-1:2.45.0-1.el9.noarch 44/99 2026-02-22T21:35:26.335 INFO:teuthology.orchestra.run.trial026.stdout: Installing : python3-kubernetes-1:26.1.0-3.el9.noarch 45/99 2026-02-22T21:35:26.338 INFO:teuthology.orchestra.run.trial026.stdout: Installing : python3-backports-tarfile-1.2.0-1.el9.noarch 46/99 2026-02-22T21:35:26.344 INFO:teuthology.orchestra.run.trial026.stdout: Installing : python3-jaraco-context-6.0.1-3.el9.noarch 47/99 2026-02-22T21:35:26.346 INFO:teuthology.orchestra.run.trial026.stdout: Installing : python3-autocommand-2.2.2-8.el9.noarch 48/99 2026-02-22T21:35:26.374 INFO:teuthology.orchestra.run.trial026.stdout: Installing : grpc-data-1.46.7-10.el9.noarch 49/99 2026-02-22T21:35:26.418 INFO:teuthology.orchestra.run.trial026.stdout: Installing : abseil-cpp-20211102.0-4.el9.x86_64 50/99 2026-02-22T21:35:26.425 INFO:teuthology.orchestra.run.trial026.stdout: Installing : python3-grpcio-1.46.7-10.el9.x86_64 51/99 2026-02-22T21:35:26.428 INFO:teuthology.orchestra.run.trial026.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 52/99 2026-02-22T21:35:26.434 INFO:teuthology.orchestra.run.trial026.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 53/99 2026-02-22T21:35:26.438 INFO:teuthology.orchestra.run.trial026.stdout: Installing : python3-jaraco-text-4.0.0-2.el9.noarch 54/99 2026-02-22T21:35:26.445 INFO:teuthology.orchestra.run.trial026.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 55/99 2026-02-22T21:35:26.448 INFO:teuthology.orchestra.run.trial026.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 56/99 2026-02-22T21:35:26.476 INFO:teuthology.orchestra.run.trial026.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 57/99 2026-02-22T21:35:26.486 INFO:teuthology.orchestra.run.trial026.stdout: Installing : python3-protobuf-3.14.0-17.el9.noarch 58/99 2026-02-22T21:35:26.521 INFO:teuthology.orchestra.run.trial026.stdout: Installing : python3-grpcio-tools-1.46.7-10.el9.x86_64 59/99 2026-02-22T21:35:26.747 INFO:teuthology.orchestra.run.trial026.stdout: Installing : python3-devel-3.9.25-3.el9.x86_64 60/99 2026-02-22T21:35:26.770 INFO:teuthology.orchestra.run.trial026.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 61/99 2026-02-22T21:35:26.774 INFO:teuthology.orchestra.run.trial026.stdout: Installing : python3-jinja2-2.11.3-8.el9.noarch 62/99 2026-02-22T21:35:26.833 INFO:teuthology.orchestra.run.trial026.stdout: Installing : openblas-0.3.29-1.el9.x86_64 63/99 2026-02-22T21:35:26.835 INFO:teuthology.orchestra.run.trial026.stdout: Installing : openblas-openmp-0.3.29-1.el9.x86_64 64/99 2026-02-22T21:35:26.856 INFO:teuthology.orchestra.run.trial026.stdout: Installing : flexiblas-openblas-openmp-3.0.4-9.el9.x86_64 65/99 2026-02-22T21:35:27.184 INFO:teuthology.orchestra.run.trial026.stdout: Installing : flexiblas-netlib-3.0.4-9.el9.x86_64 66/99 2026-02-22T21:35:27.258 INFO:teuthology.orchestra.run.trial026.stdout: Installing : python3-numpy-1:1.23.5-2.el9.x86_64 67/99 2026-02-22T21:35:27.933 INFO:teuthology.orchestra.run.trial026.stdout: Installing : python3-numpy-f2py-1:1.23.5-2.el9.x86_64 68/99 2026-02-22T21:35:27.950 INFO:teuthology.orchestra.run.trial026.stdout: Installing : python3-scipy-1.9.3-2.el9.x86_64 69/99 2026-02-22T21:35:27.960 INFO:teuthology.orchestra.run.trial026.stdout: Installing : python3-pyparsing-2.4.7-9.el9.noarch 70/99 2026-02-22T21:35:27.966 INFO:teuthology.orchestra.run.trial026.stdout: Installing : python3-packaging-20.9-5.el9.noarch 71/99 2026-02-22T21:35:27.980 INFO:teuthology.orchestra.run.trial026.stdout: Installing : python3-ply-3.11-14.el9.noarch 72/99 2026-02-22T21:35:27.996 INFO:teuthology.orchestra.run.trial026.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 73/99 2026-02-22T21:35:28.072 INFO:teuthology.orchestra.run.trial026.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 74/99 2026-02-22T21:35:28.081 INFO:teuthology.orchestra.run.trial026.stdout: Installing : python3-cryptography-36.0.1-5.el9.x86_64 75/99 2026-02-22T21:35:28.105 INFO:teuthology.orchestra.run.trial026.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 76/99 2026-02-22T21:35:28.134 INFO:teuthology.orchestra.run.trial026.stdout: Installing : python3-cheroot-10.0.1-4.el9.noarch 77/99 2026-02-22T21:35:28.185 INFO:teuthology.orchestra.run.trial026.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 78/99 2026-02-22T21:35:28.191 INFO:teuthology.orchestra.run.trial026.stdout: Installing : python3-asyncssh-2.13.2-5.el9.noarch 79/99 2026-02-22T21:35:28.195 INFO:teuthology.orchestra.run.trial026.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 80/99 2026-02-22T21:35:28.196 INFO:teuthology.orchestra.run.trial026.stdout: Installing : libconfig-1.7.2-9.el9.x86_64 81/99 2026-02-22T21:35:28.214 INFO:teuthology.orchestra.run.trial026.stdout: Running scriptlet: libstoragemgmt-1.10.1-1.el9.x86_64 82/99 2026-02-22T21:35:28.214 INFO:teuthology.orchestra.run.trial026.stdout:Creating group 'libstoragemgmt' with GID 991. 2026-02-22T21:35:28.214 INFO:teuthology.orchestra.run.trial026.stdout:Creating user 'libstoragemgmt' (daemon account for libstoragemgmt) with UID 991 and GID 991. 2026-02-22T21:35:28.215 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:35:28.222 INFO:teuthology.orchestra.run.trial026.stdout: Installing : libstoragemgmt-1.10.1-1.el9.x86_64 82/99 2026-02-22T21:35:28.246 INFO:teuthology.orchestra.run.trial026.stdout: Running scriptlet: libstoragemgmt-1.10.1-1.el9.x86_64 82/99 2026-02-22T21:35:28.246 INFO:teuthology.orchestra.run.trial026.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/libstoragemgmt.service → /usr/lib/systemd/system/libstoragemgmt.service. 2026-02-22T21:35:28.246 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:35:28.282 INFO:teuthology.orchestra.run.trial026.stdout: Installing : python3-libstoragemgmt-1.10.1-1.el9.x86_64 83/99 2026-02-22T21:35:28.346 INFO:teuthology.orchestra.run.trial026.stdout: Installing : cryptsetup-2.8.1-3.el9.x86_64 84/99 2026-02-22T21:35:28.349 INFO:teuthology.orchestra.run.trial026.stdout: Installing : ceph-volume-2:19.2.3-689.gf90f4eae.el9.noarch 85/99 2026-02-22T21:35:28.362 INFO:teuthology.orchestra.run.trial026.stdout: Running scriptlet: ceph-volume-2:19.2.3-689.gf90f4eae.el9.noarch 85/99 2026-02-22T21:35:28.363 INFO:teuthology.orchestra.run.trial026.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-22T21:35:28.363 INFO:teuthology.orchestra.run.trial026.stdout:Invalid unit name "ceph-volume@*.service" escaped as "ceph-volume@\x2a.service". 2026-02-22T21:35:28.363 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:35:29.058 INFO:teuthology.orchestra.run.trial026.stdout: Installing : ceph-osd-2:19.2.3-689.gf90f4eae.el9.x86_64 86/99 2026-02-22T21:35:29.083 INFO:teuthology.orchestra.run.trial026.stdout: Running scriptlet: ceph-osd-2:19.2.3-689.gf90f4eae.el9.x86_64 86/99 2026-02-22T21:35:29.083 INFO:teuthology.orchestra.run.trial026.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-22T21:35:29.083 INFO:teuthology.orchestra.run.trial026.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2026-02-22T21:35:29.084 INFO:teuthology.orchestra.run.trial026.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2026-02-22T21:35:29.084 INFO:teuthology.orchestra.run.trial026.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2026-02-22T21:35:29.084 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:35:29.144 INFO:teuthology.orchestra.run.trial026.stdout: Running scriptlet: cephadm-2:19.2.3-689.gf90f4eae.el9.noarch 87/99 2026-02-22T21:35:29.146 INFO:teuthology.orchestra.run.trial026.stdout: Installing : cephadm-2:19.2.3-689.gf90f4eae.el9.noarch 87/99 2026-02-22T21:35:29.151 INFO:teuthology.orchestra.run.trial026.stdout: Installing : ceph-prometheus-alerts-2:19.2.3-689.gf90f4eae.el9. 88/99 2026-02-22T21:35:29.170 INFO:teuthology.orchestra.run.trial026.stdout: Installing : ceph-grafana-dashboards-2:19.2.3-689.gf90f4eae.el9 89/99 2026-02-22T21:35:29.172 INFO:teuthology.orchestra.run.trial026.stdout: Installing : ceph-mgr-cephadm-2:19.2.3-689.gf90f4eae.el9.noarch 90/99 2026-02-22T21:35:29.664 INFO:teuthology.orchestra.run.trial026.stdout: Running scriptlet: ceph-mgr-cephadm-2:19.2.3-689.gf90f4eae.el9.noarch 90/99 2026-02-22T21:35:29.667 INFO:teuthology.orchestra.run.trial026.stdout: Installing : ceph-mgr-dashboard-2:19.2.3-689.gf90f4eae.el9.noar 91/99 2026-02-22T21:35:30.120 INFO:teuthology.orchestra.run.trial026.stdout: Running scriptlet: ceph-mgr-dashboard-2:19.2.3-689.gf90f4eae.el9.noar 91/99 2026-02-22T21:35:30.121 INFO:teuthology.orchestra.run.trial026.stdout: Installing : ceph-mgr-diskprediction-local-2:19.2.3-689.gf90f4e 92/99 2026-02-22T21:35:30.134 INFO:teuthology.orchestra.run.trial026.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:19.2.3-689.gf90f4e 92/99 2026-02-22T21:35:30.135 INFO:teuthology.orchestra.run.trial026.stdout: Installing : ceph-mgr-k8sevents-2:19.2.3-689.gf90f4eae.el9.noar 93/99 2026-02-22T21:35:30.190 INFO:teuthology.orchestra.run.trial026.stdout: Running scriptlet: ceph-mgr-k8sevents-2:19.2.3-689.gf90f4eae.el9.noar 93/99 2026-02-22T21:35:30.239 INFO:teuthology.orchestra.run.trial026.stdout: Installing : ceph-mgr-modules-core-2:19.2.3-689.gf90f4eae.el9.n 94/99 2026-02-22T21:35:30.240 INFO:teuthology.orchestra.run.trial026.stdout: Installing : ceph-mgr-2:19.2.3-689.gf90f4eae.el9.x86_64 95/99 2026-02-22T21:35:30.261 INFO:teuthology.orchestra.run.trial026.stdout: Running scriptlet: ceph-mgr-2:19.2.3-689.gf90f4eae.el9.x86_64 95/99 2026-02-22T21:35:30.262 INFO:teuthology.orchestra.run.trial026.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-22T21:35:30.262 INFO:teuthology.orchestra.run.trial026.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2026-02-22T21:35:30.262 INFO:teuthology.orchestra.run.trial026.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2026-02-22T21:35:30.262 INFO:teuthology.orchestra.run.trial026.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2026-02-22T21:35:30.262 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:35:30.273 INFO:teuthology.orchestra.run.trial026.stdout: Installing : ceph-mgr-rook-2:19.2.3-689.gf90f4eae.el9.noarch 96/99 2026-02-22T21:35:30.489 INFO:teuthology.orchestra.run.trial026.stdout: Running scriptlet: ceph-mgr-rook-2:19.2.3-689.gf90f4eae.el9.noarch 96/99 2026-02-22T21:35:30.490 INFO:teuthology.orchestra.run.trial026.stdout: Installing : ceph-mon-2:19.2.3-689.gf90f4eae.el9.x86_64 97/99 2026-02-22T21:35:30.512 INFO:teuthology.orchestra.run.trial026.stdout: Running scriptlet: ceph-mon-2:19.2.3-689.gf90f4eae.el9.x86_64 97/99 2026-02-22T21:35:30.512 INFO:teuthology.orchestra.run.trial026.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-22T21:35:30.512 INFO:teuthology.orchestra.run.trial026.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2026-02-22T21:35:30.512 INFO:teuthology.orchestra.run.trial026.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2026-02-22T21:35:30.512 INFO:teuthology.orchestra.run.trial026.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2026-02-22T21:35:30.512 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:35:30.614 INFO:teuthology.orchestra.run.trial026.stdout: Installing : ceph-mds-2:19.2.3-689.gf90f4eae.el9.x86_64 98/99 2026-02-22T21:35:30.636 INFO:teuthology.orchestra.run.trial026.stdout: Running scriptlet: ceph-mds-2:19.2.3-689.gf90f4eae.el9.x86_64 98/99 2026-02-22T21:35:30.636 INFO:teuthology.orchestra.run.trial026.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-22T21:35:30.636 INFO:teuthology.orchestra.run.trial026.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2026-02-22T21:35:30.636 INFO:teuthology.orchestra.run.trial026.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2026-02-22T21:35:30.636 INFO:teuthology.orchestra.run.trial026.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2026-02-22T21:35:30.636 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:35:30.655 INFO:teuthology.orchestra.run.trial026.stdout: Installing : ceph-2:19.2.3-689.gf90f4eae.el9.x86_64 99/99 2026-02-22T21:35:31.304 INFO:teuthology.orchestra.run.trial026.stdout: Running scriptlet: ceph-2:19.2.3-689.gf90f4eae.el9.x86_64 99/99 2026-02-22T21:35:31.304 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : ceph-2:19.2.3-689.gf90f4eae.el9.x86_64 1/99 2026-02-22T21:35:31.304 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : ceph-mds-2:19.2.3-689.gf90f4eae.el9.x86_64 2/99 2026-02-22T21:35:31.304 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : ceph-mgr-2:19.2.3-689.gf90f4eae.el9.x86_64 3/99 2026-02-22T21:35:31.304 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : ceph-mon-2:19.2.3-689.gf90f4eae.el9.x86_64 4/99 2026-02-22T21:35:31.304 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : ceph-osd-2:19.2.3-689.gf90f4eae.el9.x86_64 5/99 2026-02-22T21:35:31.304 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : ceph-grafana-dashboards-2:19.2.3-689.gf90f4eae.el9 6/99 2026-02-22T21:35:31.304 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : ceph-mgr-cephadm-2:19.2.3-689.gf90f4eae.el9.noarch 7/99 2026-02-22T21:35:31.304 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : ceph-mgr-dashboard-2:19.2.3-689.gf90f4eae.el9.noar 8/99 2026-02-22T21:35:31.304 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : ceph-mgr-diskprediction-local-2:19.2.3-689.gf90f4e 9/99 2026-02-22T21:35:31.304 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : ceph-mgr-k8sevents-2:19.2.3-689.gf90f4eae.el9.noar 10/99 2026-02-22T21:35:31.304 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : ceph-mgr-modules-core-2:19.2.3-689.gf90f4eae.el9.n 11/99 2026-02-22T21:35:31.304 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : ceph-mgr-rook-2:19.2.3-689.gf90f4eae.el9.noarch 12/99 2026-02-22T21:35:31.305 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : ceph-prometheus-alerts-2:19.2.3-689.gf90f4eae.el9. 13/99 2026-02-22T21:35:31.305 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : ceph-volume-2:19.2.3-689.gf90f4eae.el9.noarch 14/99 2026-02-22T21:35:31.305 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : cephadm-2:19.2.3-689.gf90f4eae.el9.noarch 15/99 2026-02-22T21:35:31.305 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : cryptsetup-2.8.1-3.el9.x86_64 16/99 2026-02-22T21:35:31.305 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : ledmon-libs-1.1.0-3.el9.x86_64 17/99 2026-02-22T21:35:31.305 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : libconfig-1.7.2-9.el9.x86_64 18/99 2026-02-22T21:35:31.305 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : libgfortran-11.5.0-14.el9.x86_64 19/99 2026-02-22T21:35:31.305 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : libquadmath-11.5.0-14.el9.x86_64 20/99 2026-02-22T21:35:31.305 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 21/99 2026-02-22T21:35:31.305 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : python3-cryptography-36.0.1-5.el9.x86_64 22/99 2026-02-22T21:35:31.305 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : python3-ply-3.11-14.el9.noarch 23/99 2026-02-22T21:35:31.305 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 24/99 2026-02-22T21:35:31.305 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : python3-pyparsing-2.4.7-9.el9.noarch 25/99 2026-02-22T21:35:31.305 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : python3-requests-2.25.1-10.el9.noarch 26/99 2026-02-22T21:35:31.305 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : python3-urllib3-1.26.5-7.el9.noarch 27/99 2026-02-22T21:35:31.305 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : unzip-6.0-59.el9.x86_64 28/99 2026-02-22T21:35:31.306 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : zip-3.0-35.el9.x86_64 29/99 2026-02-22T21:35:31.306 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : flexiblas-3.0.4-9.el9.x86_64 30/99 2026-02-22T21:35:31.306 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : flexiblas-netlib-3.0.4-9.el9.x86_64 31/99 2026-02-22T21:35:31.306 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-9.el9.x86_64 32/99 2026-02-22T21:35:31.306 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : libstoragemgmt-1.10.1-1.el9.x86_64 33/99 2026-02-22T21:35:31.306 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : lua-5.4.4-4.el9.x86_64 34/99 2026-02-22T21:35:31.306 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : openblas-0.3.29-1.el9.x86_64 35/99 2026-02-22T21:35:31.306 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : openblas-openmp-0.3.29-1.el9.x86_64 36/99 2026-02-22T21:35:31.306 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : protobuf-3.14.0-17.el9.x86_64 37/99 2026-02-22T21:35:31.306 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 38/99 2026-02-22T21:35:31.306 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : python3-devel-3.9.25-3.el9.x86_64 39/99 2026-02-22T21:35:31.306 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : python3-jinja2-2.11.3-8.el9.noarch 40/99 2026-02-22T21:35:31.306 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : python3-libstoragemgmt-1.10.1-1.el9.x86_64 41/99 2026-02-22T21:35:31.307 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : python3-lxml-4.6.5-3.el9.x86_64 42/99 2026-02-22T21:35:31.307 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : python3-mako-1.1.4-6.el9.noarch 43/99 2026-02-22T21:35:31.307 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 44/99 2026-02-22T21:35:31.307 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : python3-numpy-1:1.23.5-2.el9.x86_64 45/99 2026-02-22T21:35:31.307 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : python3-numpy-f2py-1:1.23.5-2.el9.x86_64 46/99 2026-02-22T21:35:31.307 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : python3-packaging-20.9-5.el9.noarch 47/99 2026-02-22T21:35:31.307 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : python3-protobuf-3.14.0-17.el9.noarch 48/99 2026-02-22T21:35:31.307 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : python3-pyasn1-0.4.8-6.el9.noarch 49/99 2026-02-22T21:35:31.307 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : python3-pyasn1-modules-0.4.8-6.el9.noarch 50/99 2026-02-22T21:35:31.307 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 51/99 2026-02-22T21:35:31.307 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : python3-scipy-1.9.3-2.el9.x86_64 52/99 2026-02-22T21:35:31.307 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 53/99 2026-02-22T21:35:31.308 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : xmlsec1-1.2.29-13.el9.x86_64 54/99 2026-02-22T21:35:31.308 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : xmlsec1-openssl-1.2.29-13.el9.x86_64 55/99 2026-02-22T21:35:31.308 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : lua-devel-5.4.4-4.el9.x86_64 56/99 2026-02-22T21:35:31.308 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : protobuf-compiler-3.14.0-17.el9.x86_64 57/99 2026-02-22T21:35:31.308 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : abseil-cpp-20211102.0-4.el9.x86_64 58/99 2026-02-22T21:35:31.308 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : grpc-data-1.46.7-10.el9.noarch 59/99 2026-02-22T21:35:31.308 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : luarocks-3.9.2-5.el9.noarch 60/99 2026-02-22T21:35:31.308 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : python3-asyncssh-2.13.2-5.el9.noarch 61/99 2026-02-22T21:35:31.308 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : python3-autocommand-2.2.2-8.el9.noarch 62/99 2026-02-22T21:35:31.308 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : python3-backports-tarfile-1.2.0-1.el9.noarch 63/99 2026-02-22T21:35:31.308 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 64/99 2026-02-22T21:35:31.308 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 65/99 2026-02-22T21:35:31.308 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : python3-certifi-2023.05.07-4.el9.noarch 66/99 2026-02-22T21:35:31.309 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : python3-cheroot-10.0.1-4.el9.noarch 67/99 2026-02-22T21:35:31.309 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 68/99 2026-02-22T21:35:31.309 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : python3-google-auth-1:2.45.0-1.el9.noarch 69/99 2026-02-22T21:35:31.309 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : python3-grpcio-1.46.7-10.el9.x86_64 70/99 2026-02-22T21:35:31.309 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : python3-grpcio-tools-1.46.7-10.el9.x86_64 71/99 2026-02-22T21:35:31.309 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : python3-influxdb-5.3.1-1.el9.noarch 72/99 2026-02-22T21:35:31.309 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : python3-isodate-0.6.1-3.el9.noarch 73/99 2026-02-22T21:35:31.309 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 74/99 2026-02-22T21:35:31.309 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 75/99 2026-02-22T21:35:31.309 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 76/99 2026-02-22T21:35:31.309 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : python3-jaraco-context-6.0.1-3.el9.noarch 77/99 2026-02-22T21:35:31.309 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 78/99 2026-02-22T21:35:31.309 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : python3-jaraco-text-4.0.0-2.el9.noarch 79/99 2026-02-22T21:35:31.310 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : python3-kubernetes-1:26.1.0-3.el9.noarch 80/99 2026-02-22T21:35:31.310 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : python3-logutils-0.3.5-21.el9.noarch 81/99 2026-02-22T21:35:31.310 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 82/99 2026-02-22T21:35:31.310 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : python3-msgpack-1.0.3-2.el9.x86_64 83/99 2026-02-22T21:35:31.310 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 84/99 2026-02-22T21:35:31.310 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : python3-pecan-1.4.2-3.el9.noarch 85/99 2026-02-22T21:35:31.310 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 86/99 2026-02-22T21:35:31.310 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 87/99 2026-02-22T21:35:31.310 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 88/99 2026-02-22T21:35:31.310 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 89/99 2026-02-22T21:35:31.310 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 90/99 2026-02-22T21:35:31.310 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : python3-saml-1.16.0-1.el9.noarch 91/99 2026-02-22T21:35:31.310 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 92/99 2026-02-22T21:35:31.311 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : python3-typing-extensions-4.15.0-1.el9.noarch 93/99 2026-02-22T21:35:31.311 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : python3-webob-1.8.8-2.el9.noarch 94/99 2026-02-22T21:35:31.311 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 95/99 2026-02-22T21:35:31.311 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : python3-werkzeug-2.0.3-3.el9.1.noarch 96/99 2026-02-22T21:35:31.311 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : python3-xmlsec-1.3.13-1.el9.x86_64 97/99 2026-02-22T21:35:31.311 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : python3-xmltodict-0.12.0-15.el9.noarch 98/99 2026-02-22T21:35:31.372 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 99/99 2026-02-22T21:35:31.372 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:35:31.372 INFO:teuthology.orchestra.run.trial026.stdout:Installed: 2026-02-22T21:35:31.372 INFO:teuthology.orchestra.run.trial026.stdout: abseil-cpp-20211102.0-4.el9.x86_64 2026-02-22T21:35:31.372 INFO:teuthology.orchestra.run.trial026.stdout: ceph-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:35:31.372 INFO:teuthology.orchestra.run.trial026.stdout: ceph-grafana-dashboards-2:19.2.3-689.gf90f4eae.el9.noarch 2026-02-22T21:35:31.372 INFO:teuthology.orchestra.run.trial026.stdout: ceph-mds-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:35:31.372 INFO:teuthology.orchestra.run.trial026.stdout: ceph-mgr-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:35:31.372 INFO:teuthology.orchestra.run.trial026.stdout: ceph-mgr-cephadm-2:19.2.3-689.gf90f4eae.el9.noarch 2026-02-22T21:35:31.372 INFO:teuthology.orchestra.run.trial026.stdout: ceph-mgr-dashboard-2:19.2.3-689.gf90f4eae.el9.noarch 2026-02-22T21:35:31.372 INFO:teuthology.orchestra.run.trial026.stdout: ceph-mgr-diskprediction-local-2:19.2.3-689.gf90f4eae.el9.noarch 2026-02-22T21:35:31.372 INFO:teuthology.orchestra.run.trial026.stdout: ceph-mgr-k8sevents-2:19.2.3-689.gf90f4eae.el9.noarch 2026-02-22T21:35:31.372 INFO:teuthology.orchestra.run.trial026.stdout: ceph-mgr-modules-core-2:19.2.3-689.gf90f4eae.el9.noarch 2026-02-22T21:35:31.372 INFO:teuthology.orchestra.run.trial026.stdout: ceph-mgr-rook-2:19.2.3-689.gf90f4eae.el9.noarch 2026-02-22T21:35:31.372 INFO:teuthology.orchestra.run.trial026.stdout: ceph-mon-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:35:31.372 INFO:teuthology.orchestra.run.trial026.stdout: ceph-osd-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:35:31.372 INFO:teuthology.orchestra.run.trial026.stdout: ceph-prometheus-alerts-2:19.2.3-689.gf90f4eae.el9.noarch 2026-02-22T21:35:31.372 INFO:teuthology.orchestra.run.trial026.stdout: ceph-volume-2:19.2.3-689.gf90f4eae.el9.noarch 2026-02-22T21:35:31.373 INFO:teuthology.orchestra.run.trial026.stdout: cephadm-2:19.2.3-689.gf90f4eae.el9.noarch 2026-02-22T21:35:31.373 INFO:teuthology.orchestra.run.trial026.stdout: cryptsetup-2.8.1-3.el9.x86_64 2026-02-22T21:35:31.373 INFO:teuthology.orchestra.run.trial026.stdout: flexiblas-3.0.4-9.el9.x86_64 2026-02-22T21:35:31.373 INFO:teuthology.orchestra.run.trial026.stdout: flexiblas-netlib-3.0.4-9.el9.x86_64 2026-02-22T21:35:31.373 INFO:teuthology.orchestra.run.trial026.stdout: flexiblas-openblas-openmp-3.0.4-9.el9.x86_64 2026-02-22T21:35:31.373 INFO:teuthology.orchestra.run.trial026.stdout: grpc-data-1.46.7-10.el9.noarch 2026-02-22T21:35:31.373 INFO:teuthology.orchestra.run.trial026.stdout: ledmon-libs-1.1.0-3.el9.x86_64 2026-02-22T21:35:31.373 INFO:teuthology.orchestra.run.trial026.stdout: libconfig-1.7.2-9.el9.x86_64 2026-02-22T21:35:31.373 INFO:teuthology.orchestra.run.trial026.stdout: libgfortran-11.5.0-14.el9.x86_64 2026-02-22T21:35:31.373 INFO:teuthology.orchestra.run.trial026.stdout: libquadmath-11.5.0-14.el9.x86_64 2026-02-22T21:35:31.373 INFO:teuthology.orchestra.run.trial026.stdout: libstoragemgmt-1.10.1-1.el9.x86_64 2026-02-22T21:35:31.373 INFO:teuthology.orchestra.run.trial026.stdout: lua-5.4.4-4.el9.x86_64 2026-02-22T21:35:31.373 INFO:teuthology.orchestra.run.trial026.stdout: lua-devel-5.4.4-4.el9.x86_64 2026-02-22T21:35:31.373 INFO:teuthology.orchestra.run.trial026.stdout: luarocks-3.9.2-5.el9.noarch 2026-02-22T21:35:31.373 INFO:teuthology.orchestra.run.trial026.stdout: openblas-0.3.29-1.el9.x86_64 2026-02-22T21:35:31.373 INFO:teuthology.orchestra.run.trial026.stdout: openblas-openmp-0.3.29-1.el9.x86_64 2026-02-22T21:35:31.373 INFO:teuthology.orchestra.run.trial026.stdout: protobuf-3.14.0-17.el9.x86_64 2026-02-22T21:35:31.373 INFO:teuthology.orchestra.run.trial026.stdout: protobuf-compiler-3.14.0-17.el9.x86_64 2026-02-22T21:35:31.373 INFO:teuthology.orchestra.run.trial026.stdout: python3-asyncssh-2.13.2-5.el9.noarch 2026-02-22T21:35:31.373 INFO:teuthology.orchestra.run.trial026.stdout: python3-autocommand-2.2.2-8.el9.noarch 2026-02-22T21:35:31.373 INFO:teuthology.orchestra.run.trial026.stdout: python3-babel-2.9.1-2.el9.noarch 2026-02-22T21:35:31.373 INFO:teuthology.orchestra.run.trial026.stdout: python3-backports-tarfile-1.2.0-1.el9.noarch 2026-02-22T21:35:31.373 INFO:teuthology.orchestra.run.trial026.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2026-02-22T21:35:31.373 INFO:teuthology.orchestra.run.trial026.stdout: python3-cachetools-4.2.4-1.el9.noarch 2026-02-22T21:35:31.373 INFO:teuthology.orchestra.run.trial026.stdout: python3-certifi-2023.05.07-4.el9.noarch 2026-02-22T21:35:31.373 INFO:teuthology.orchestra.run.trial026.stdout: python3-cffi-1.14.5-5.el9.x86_64 2026-02-22T21:35:31.373 INFO:teuthology.orchestra.run.trial026.stdout: python3-cheroot-10.0.1-4.el9.noarch 2026-02-22T21:35:31.373 INFO:teuthology.orchestra.run.trial026.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2026-02-22T21:35:31.373 INFO:teuthology.orchestra.run.trial026.stdout: python3-cryptography-36.0.1-5.el9.x86_64 2026-02-22T21:35:31.373 INFO:teuthology.orchestra.run.trial026.stdout: python3-devel-3.9.25-3.el9.x86_64 2026-02-22T21:35:31.373 INFO:teuthology.orchestra.run.trial026.stdout: python3-google-auth-1:2.45.0-1.el9.noarch 2026-02-22T21:35:31.373 INFO:teuthology.orchestra.run.trial026.stdout: python3-grpcio-1.46.7-10.el9.x86_64 2026-02-22T21:35:31.373 INFO:teuthology.orchestra.run.trial026.stdout: python3-grpcio-tools-1.46.7-10.el9.x86_64 2026-02-22T21:35:31.373 INFO:teuthology.orchestra.run.trial026.stdout: python3-influxdb-5.3.1-1.el9.noarch 2026-02-22T21:35:31.373 INFO:teuthology.orchestra.run.trial026.stdout: python3-isodate-0.6.1-3.el9.noarch 2026-02-22T21:35:31.373 INFO:teuthology.orchestra.run.trial026.stdout: python3-jaraco-8.2.1-3.el9.noarch 2026-02-22T21:35:31.373 INFO:teuthology.orchestra.run.trial026.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2026-02-22T21:35:31.373 INFO:teuthology.orchestra.run.trial026.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2026-02-22T21:35:31.373 INFO:teuthology.orchestra.run.trial026.stdout: python3-jaraco-context-6.0.1-3.el9.noarch 2026-02-22T21:35:31.373 INFO:teuthology.orchestra.run.trial026.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2026-02-22T21:35:31.374 INFO:teuthology.orchestra.run.trial026.stdout: python3-jaraco-text-4.0.0-2.el9.noarch 2026-02-22T21:35:31.374 INFO:teuthology.orchestra.run.trial026.stdout: python3-jinja2-2.11.3-8.el9.noarch 2026-02-22T21:35:31.374 INFO:teuthology.orchestra.run.trial026.stdout: python3-kubernetes-1:26.1.0-3.el9.noarch 2026-02-22T21:35:31.374 INFO:teuthology.orchestra.run.trial026.stdout: python3-libstoragemgmt-1.10.1-1.el9.x86_64 2026-02-22T21:35:31.374 INFO:teuthology.orchestra.run.trial026.stdout: python3-logutils-0.3.5-21.el9.noarch 2026-02-22T21:35:31.374 INFO:teuthology.orchestra.run.trial026.stdout: python3-lxml-4.6.5-3.el9.x86_64 2026-02-22T21:35:31.374 INFO:teuthology.orchestra.run.trial026.stdout: python3-mako-1.1.4-6.el9.noarch 2026-02-22T21:35:31.374 INFO:teuthology.orchestra.run.trial026.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2026-02-22T21:35:31.374 INFO:teuthology.orchestra.run.trial026.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2026-02-22T21:35:31.374 INFO:teuthology.orchestra.run.trial026.stdout: python3-msgpack-1.0.3-2.el9.x86_64 2026-02-22T21:35:31.374 INFO:teuthology.orchestra.run.trial026.stdout: python3-natsort-7.1.1-5.el9.noarch 2026-02-22T21:35:31.374 INFO:teuthology.orchestra.run.trial026.stdout: python3-numpy-1:1.23.5-2.el9.x86_64 2026-02-22T21:35:31.374 INFO:teuthology.orchestra.run.trial026.stdout: python3-numpy-f2py-1:1.23.5-2.el9.x86_64 2026-02-22T21:35:31.374 INFO:teuthology.orchestra.run.trial026.stdout: python3-packaging-20.9-5.el9.noarch 2026-02-22T21:35:31.374 INFO:teuthology.orchestra.run.trial026.stdout: python3-pecan-1.4.2-3.el9.noarch 2026-02-22T21:35:31.374 INFO:teuthology.orchestra.run.trial026.stdout: python3-ply-3.11-14.el9.noarch 2026-02-22T21:35:31.374 INFO:teuthology.orchestra.run.trial026.stdout: python3-portend-3.1.0-2.el9.noarch 2026-02-22T21:35:31.374 INFO:teuthology.orchestra.run.trial026.stdout: python3-protobuf-3.14.0-17.el9.noarch 2026-02-22T21:35:31.374 INFO:teuthology.orchestra.run.trial026.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2026-02-22T21:35:31.374 INFO:teuthology.orchestra.run.trial026.stdout: python3-pyasn1-0.4.8-6.el9.noarch 2026-02-22T21:35:31.374 INFO:teuthology.orchestra.run.trial026.stdout: python3-pyasn1-modules-0.4.8-6.el9.noarch 2026-02-22T21:35:31.374 INFO:teuthology.orchestra.run.trial026.stdout: python3-pycparser-2.20-6.el9.noarch 2026-02-22T21:35:31.374 INFO:teuthology.orchestra.run.trial026.stdout: python3-pyparsing-2.4.7-9.el9.noarch 2026-02-22T21:35:31.374 INFO:teuthology.orchestra.run.trial026.stdout: python3-repoze-lru-0.7-16.el9.noarch 2026-02-22T21:35:31.374 INFO:teuthology.orchestra.run.trial026.stdout: python3-requests-2.25.1-10.el9.noarch 2026-02-22T21:35:31.374 INFO:teuthology.orchestra.run.trial026.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2026-02-22T21:35:31.374 INFO:teuthology.orchestra.run.trial026.stdout: python3-routes-2.5.1-5.el9.noarch 2026-02-22T21:35:31.374 INFO:teuthology.orchestra.run.trial026.stdout: python3-rsa-4.9-2.el9.noarch 2026-02-22T21:35:31.374 INFO:teuthology.orchestra.run.trial026.stdout: python3-saml-1.16.0-1.el9.noarch 2026-02-22T21:35:31.374 INFO:teuthology.orchestra.run.trial026.stdout: python3-scipy-1.9.3-2.el9.x86_64 2026-02-22T21:35:31.374 INFO:teuthology.orchestra.run.trial026.stdout: python3-tempora-5.0.0-2.el9.noarch 2026-02-22T21:35:31.374 INFO:teuthology.orchestra.run.trial026.stdout: python3-toml-0.10.2-6.el9.noarch 2026-02-22T21:35:31.374 INFO:teuthology.orchestra.run.trial026.stdout: python3-typing-extensions-4.15.0-1.el9.noarch 2026-02-22T21:35:31.374 INFO:teuthology.orchestra.run.trial026.stdout: python3-urllib3-1.26.5-7.el9.noarch 2026-02-22T21:35:31.374 INFO:teuthology.orchestra.run.trial026.stdout: python3-webob-1.8.8-2.el9.noarch 2026-02-22T21:35:31.374 INFO:teuthology.orchestra.run.trial026.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2026-02-22T21:35:31.374 INFO:teuthology.orchestra.run.trial026.stdout: python3-werkzeug-2.0.3-3.el9.1.noarch 2026-02-22T21:35:31.374 INFO:teuthology.orchestra.run.trial026.stdout: python3-xmlsec-1.3.13-1.el9.x86_64 2026-02-22T21:35:31.374 INFO:teuthology.orchestra.run.trial026.stdout: python3-xmltodict-0.12.0-15.el9.noarch 2026-02-22T21:35:31.374 INFO:teuthology.orchestra.run.trial026.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2026-02-22T21:35:31.374 INFO:teuthology.orchestra.run.trial026.stdout: unzip-6.0-59.el9.x86_64 2026-02-22T21:35:31.375 INFO:teuthology.orchestra.run.trial026.stdout: xmlsec1-1.2.29-13.el9.x86_64 2026-02-22T21:35:31.375 INFO:teuthology.orchestra.run.trial026.stdout: xmlsec1-openssl-1.2.29-13.el9.x86_64 2026-02-22T21:35:31.375 INFO:teuthology.orchestra.run.trial026.stdout: zip-3.0-35.el9.x86_64 2026-02-22T21:35:31.375 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:35:31.375 INFO:teuthology.orchestra.run.trial026.stdout:Complete! 2026-02-22T21:35:31.428 DEBUG:teuthology.orchestra.run.trial026:> sudo yum -y install ceph-base 2026-02-22T21:35:31.747 INFO:teuthology.orchestra.run.trial026.stdout:Last metadata expiration check: 0:00:32 ago on Sun 22 Feb 2026 09:34:59 PM UTC. 2026-02-22T21:35:31.814 INFO:teuthology.orchestra.run.trial026.stdout:Package ceph-base-2:19.2.3-689.gf90f4eae.el9.x86_64 is already installed. 2026-02-22T21:35:31.832 INFO:teuthology.orchestra.run.trial026.stdout:Dependencies resolved. 2026-02-22T21:35:31.833 INFO:teuthology.orchestra.run.trial026.stdout:Nothing to do. 2026-02-22T21:35:31.833 INFO:teuthology.orchestra.run.trial026.stdout:Complete! 2026-02-22T21:35:31.854 DEBUG:teuthology.orchestra.run.trial026:> sudo yum -y install cephadm 2026-02-22T21:35:32.164 INFO:teuthology.orchestra.run.trial026.stdout:Last metadata expiration check: 0:00:33 ago on Sun 22 Feb 2026 09:34:59 PM UTC. 2026-02-22T21:35:32.229 INFO:teuthology.orchestra.run.trial026.stdout:Package cephadm-2:19.2.3-689.gf90f4eae.el9.noarch is already installed. 2026-02-22T21:35:32.247 INFO:teuthology.orchestra.run.trial026.stdout:Dependencies resolved. 2026-02-22T21:35:32.248 INFO:teuthology.orchestra.run.trial026.stdout:Nothing to do. 2026-02-22T21:35:32.248 INFO:teuthology.orchestra.run.trial026.stdout:Complete! 2026-02-22T21:35:32.269 DEBUG:teuthology.orchestra.run.trial026:> sudo yum -y install ceph-immutable-object-cache 2026-02-22T21:35:32.583 INFO:teuthology.orchestra.run.trial026.stdout:Last metadata expiration check: 0:00:33 ago on Sun 22 Feb 2026 09:34:59 PM UTC. 2026-02-22T21:35:32.667 INFO:teuthology.orchestra.run.trial026.stdout:Dependencies resolved. 2026-02-22T21:35:32.667 INFO:teuthology.orchestra.run.trial026.stdout:================================================================================ 2026-02-22T21:35:32.668 INFO:teuthology.orchestra.run.trial026.stdout: Package Arch Version Repo Size 2026-02-22T21:35:32.668 INFO:teuthology.orchestra.run.trial026.stdout:================================================================================ 2026-02-22T21:35:32.668 INFO:teuthology.orchestra.run.trial026.stdout:Installing: 2026-02-22T21:35:32.668 INFO:teuthology.orchestra.run.trial026.stdout: ceph-immutable-object-cache x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 145 k 2026-02-22T21:35:32.668 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:35:32.668 INFO:teuthology.orchestra.run.trial026.stdout:Transaction Summary 2026-02-22T21:35:32.668 INFO:teuthology.orchestra.run.trial026.stdout:================================================================================ 2026-02-22T21:35:32.668 INFO:teuthology.orchestra.run.trial026.stdout:Install 1 Package 2026-02-22T21:35:32.668 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:35:32.668 INFO:teuthology.orchestra.run.trial026.stdout:Total download size: 145 k 2026-02-22T21:35:32.668 INFO:teuthology.orchestra.run.trial026.stdout:Installed size: 431 k 2026-02-22T21:35:32.669 INFO:teuthology.orchestra.run.trial026.stdout:Downloading Packages: 2026-02-22T21:35:32.761 INFO:teuthology.orchestra.run.trial026.stdout:ceph-immutable-object-cache-19.2.3-689.gf90f4ea 1.5 MB/s | 145 kB 00:00 2026-02-22T21:35:32.761 INFO:teuthology.orchestra.run.trial026.stdout:-------------------------------------------------------------------------------- 2026-02-22T21:35:32.762 INFO:teuthology.orchestra.run.trial026.stdout:Total 1.5 MB/s | 145 kB 00:00 2026-02-22T21:35:32.762 INFO:teuthology.orchestra.run.trial026.stdout:Running transaction check 2026-02-22T21:35:32.766 INFO:teuthology.orchestra.run.trial026.stdout:Transaction check succeeded. 2026-02-22T21:35:32.766 INFO:teuthology.orchestra.run.trial026.stdout:Running transaction test 2026-02-22T21:35:32.799 INFO:teuthology.orchestra.run.trial026.stdout:Transaction test succeeded. 2026-02-22T21:35:32.799 INFO:teuthology.orchestra.run.trial026.stdout:Running transaction 2026-02-22T21:35:32.854 INFO:teuthology.orchestra.run.trial026.stdout: Preparing : 1/1 2026-02-22T21:35:32.856 INFO:teuthology.orchestra.run.trial026.stdout: Installing : ceph-immutable-object-cache-2:19.2.3-689.gf90f4eae.e 1/1 2026-02-22T21:35:32.879 INFO:teuthology.orchestra.run.trial026.stdout: Running scriptlet: ceph-immutable-object-cache-2:19.2.3-689.gf90f4eae.e 1/1 2026-02-22T21:35:32.879 INFO:teuthology.orchestra.run.trial026.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-22T21:35:32.879 INFO:teuthology.orchestra.run.trial026.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2026-02-22T21:35:32.879 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:35:33.194 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : ceph-immutable-object-cache-2:19.2.3-689.gf90f4eae.e 1/1 2026-02-22T21:35:33.194 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:35:33.194 INFO:teuthology.orchestra.run.trial026.stdout:Installed: 2026-02-22T21:35:33.195 INFO:teuthology.orchestra.run.trial026.stdout: ceph-immutable-object-cache-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:35:33.195 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:35:33.195 INFO:teuthology.orchestra.run.trial026.stdout:Complete! 2026-02-22T21:35:33.220 DEBUG:teuthology.orchestra.run.trial026:> sudo yum -y install ceph-mgr 2026-02-22T21:35:33.536 INFO:teuthology.orchestra.run.trial026.stdout:Last metadata expiration check: 0:00:34 ago on Sun 22 Feb 2026 09:34:59 PM UTC. 2026-02-22T21:35:33.599 INFO:teuthology.orchestra.run.trial026.stdout:Package ceph-mgr-2:19.2.3-689.gf90f4eae.el9.x86_64 is already installed. 2026-02-22T21:35:33.618 INFO:teuthology.orchestra.run.trial026.stdout:Dependencies resolved. 2026-02-22T21:35:33.618 INFO:teuthology.orchestra.run.trial026.stdout:Nothing to do. 2026-02-22T21:35:33.618 INFO:teuthology.orchestra.run.trial026.stdout:Complete! 2026-02-22T21:35:33.639 DEBUG:teuthology.orchestra.run.trial026:> sudo yum -y install ceph-mgr-dashboard 2026-02-22T21:35:33.951 INFO:teuthology.orchestra.run.trial026.stdout:Last metadata expiration check: 0:00:34 ago on Sun 22 Feb 2026 09:34:59 PM UTC. 2026-02-22T21:35:34.016 INFO:teuthology.orchestra.run.trial026.stdout:Package ceph-mgr-dashboard-2:19.2.3-689.gf90f4eae.el9.noarch is already installed. 2026-02-22T21:35:34.034 INFO:teuthology.orchestra.run.trial026.stdout:Dependencies resolved. 2026-02-22T21:35:34.035 INFO:teuthology.orchestra.run.trial026.stdout:Nothing to do. 2026-02-22T21:35:34.035 INFO:teuthology.orchestra.run.trial026.stdout:Complete! 2026-02-22T21:35:34.056 DEBUG:teuthology.orchestra.run.trial026:> sudo yum -y install ceph-mgr-diskprediction-local 2026-02-22T21:35:34.370 INFO:teuthology.orchestra.run.trial026.stdout:Last metadata expiration check: 0:00:35 ago on Sun 22 Feb 2026 09:34:59 PM UTC. 2026-02-22T21:35:34.437 INFO:teuthology.orchestra.run.trial026.stdout:Package ceph-mgr-diskprediction-local-2:19.2.3-689.gf90f4eae.el9.noarch is already installed. 2026-02-22T21:35:34.456 INFO:teuthology.orchestra.run.trial026.stdout:Dependencies resolved. 2026-02-22T21:35:34.456 INFO:teuthology.orchestra.run.trial026.stdout:Nothing to do. 2026-02-22T21:35:34.456 INFO:teuthology.orchestra.run.trial026.stdout:Complete! 2026-02-22T21:35:34.477 DEBUG:teuthology.orchestra.run.trial026:> sudo yum -y install ceph-mgr-rook 2026-02-22T21:35:34.792 INFO:teuthology.orchestra.run.trial026.stdout:Last metadata expiration check: 0:00:35 ago on Sun 22 Feb 2026 09:34:59 PM UTC. 2026-02-22T21:35:34.857 INFO:teuthology.orchestra.run.trial026.stdout:Package ceph-mgr-rook-2:19.2.3-689.gf90f4eae.el9.noarch is already installed. 2026-02-22T21:35:34.875 INFO:teuthology.orchestra.run.trial026.stdout:Dependencies resolved. 2026-02-22T21:35:34.876 INFO:teuthology.orchestra.run.trial026.stdout:Nothing to do. 2026-02-22T21:35:34.876 INFO:teuthology.orchestra.run.trial026.stdout:Complete! 2026-02-22T21:35:34.897 DEBUG:teuthology.orchestra.run.trial026:> sudo yum -y install ceph-mgr-cephadm 2026-02-22T21:35:35.209 INFO:teuthology.orchestra.run.trial026.stdout:Last metadata expiration check: 0:00:36 ago on Sun 22 Feb 2026 09:34:59 PM UTC. 2026-02-22T21:35:35.273 INFO:teuthology.orchestra.run.trial026.stdout:Package ceph-mgr-cephadm-2:19.2.3-689.gf90f4eae.el9.noarch is already installed. 2026-02-22T21:35:35.291 INFO:teuthology.orchestra.run.trial026.stdout:Dependencies resolved. 2026-02-22T21:35:35.292 INFO:teuthology.orchestra.run.trial026.stdout:Nothing to do. 2026-02-22T21:35:35.292 INFO:teuthology.orchestra.run.trial026.stdout:Complete! 2026-02-22T21:35:35.313 DEBUG:teuthology.orchestra.run.trial026:> sudo yum -y install ceph-fuse 2026-02-22T21:35:35.625 INFO:teuthology.orchestra.run.trial026.stdout:Last metadata expiration check: 0:00:36 ago on Sun 22 Feb 2026 09:34:59 PM UTC. 2026-02-22T21:35:35.710 INFO:teuthology.orchestra.run.trial026.stdout:Dependencies resolved. 2026-02-22T21:35:35.710 INFO:teuthology.orchestra.run.trial026.stdout:================================================================================ 2026-02-22T21:35:35.710 INFO:teuthology.orchestra.run.trial026.stdout: Package Architecture Version Repository Size 2026-02-22T21:35:35.710 INFO:teuthology.orchestra.run.trial026.stdout:================================================================================ 2026-02-22T21:35:35.710 INFO:teuthology.orchestra.run.trial026.stdout:Installing: 2026-02-22T21:35:35.710 INFO:teuthology.orchestra.run.trial026.stdout: ceph-fuse x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 1.2 M 2026-02-22T21:35:35.710 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:35:35.710 INFO:teuthology.orchestra.run.trial026.stdout:Transaction Summary 2026-02-22T21:35:35.710 INFO:teuthology.orchestra.run.trial026.stdout:================================================================================ 2026-02-22T21:35:35.710 INFO:teuthology.orchestra.run.trial026.stdout:Install 1 Package 2026-02-22T21:35:35.710 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:35:35.710 INFO:teuthology.orchestra.run.trial026.stdout:Total download size: 1.2 M 2026-02-22T21:35:35.710 INFO:teuthology.orchestra.run.trial026.stdout:Installed size: 3.6 M 2026-02-22T21:35:35.710 INFO:teuthology.orchestra.run.trial026.stdout:Downloading Packages: 2026-02-22T21:35:36.195 INFO:teuthology.orchestra.run.trial026.stdout:ceph-fuse-19.2.3-689.gf90f4eae.el9.x86_64.rpm 2.4 MB/s | 1.2 MB 00:00 2026-02-22T21:35:36.195 INFO:teuthology.orchestra.run.trial026.stdout:-------------------------------------------------------------------------------- 2026-02-22T21:35:36.195 INFO:teuthology.orchestra.run.trial026.stdout:Total 2.4 MB/s | 1.2 MB 00:00 2026-02-22T21:35:36.195 INFO:teuthology.orchestra.run.trial026.stdout:Running transaction check 2026-02-22T21:35:36.200 INFO:teuthology.orchestra.run.trial026.stdout:Transaction check succeeded. 2026-02-22T21:35:36.200 INFO:teuthology.orchestra.run.trial026.stdout:Running transaction test 2026-02-22T21:35:36.236 INFO:teuthology.orchestra.run.trial026.stdout:Transaction test succeeded. 2026-02-22T21:35:36.236 INFO:teuthology.orchestra.run.trial026.stdout:Running transaction 2026-02-22T21:35:36.339 INFO:teuthology.orchestra.run.trial026.stdout: Preparing : 1/1 2026-02-22T21:35:36.357 INFO:teuthology.orchestra.run.trial026.stdout: Installing : ceph-fuse-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T21:35:36.594 INFO:teuthology.orchestra.run.trial026.stdout: Running scriptlet: ceph-fuse-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T21:35:36.651 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : ceph-fuse-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T21:35:36.651 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:35:36.651 INFO:teuthology.orchestra.run.trial026.stdout:Installed: 2026-02-22T21:35:36.651 INFO:teuthology.orchestra.run.trial026.stdout: ceph-fuse-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:35:36.651 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:35:36.651 INFO:teuthology.orchestra.run.trial026.stdout:Complete! 2026-02-22T21:35:36.678 DEBUG:teuthology.orchestra.run.trial026:> sudo yum -y install ceph-volume 2026-02-22T21:35:36.994 INFO:teuthology.orchestra.run.trial026.stdout:Last metadata expiration check: 0:00:37 ago on Sun 22 Feb 2026 09:34:59 PM UTC. 2026-02-22T21:35:37.059 INFO:teuthology.orchestra.run.trial026.stdout:Package ceph-volume-2:19.2.3-689.gf90f4eae.el9.noarch is already installed. 2026-02-22T21:35:37.078 INFO:teuthology.orchestra.run.trial026.stdout:Dependencies resolved. 2026-02-22T21:35:37.079 INFO:teuthology.orchestra.run.trial026.stdout:Nothing to do. 2026-02-22T21:35:37.079 INFO:teuthology.orchestra.run.trial026.stdout:Complete! 2026-02-22T21:35:37.100 DEBUG:teuthology.orchestra.run.trial026:> sudo yum -y install librados-devel 2026-02-22T21:35:37.412 INFO:teuthology.orchestra.run.trial026.stdout:Last metadata expiration check: 0:00:38 ago on Sun 22 Feb 2026 09:34:59 PM UTC. 2026-02-22T21:35:37.497 INFO:teuthology.orchestra.run.trial026.stdout:Dependencies resolved. 2026-02-22T21:35:37.498 INFO:teuthology.orchestra.run.trial026.stdout:================================================================================ 2026-02-22T21:35:37.498 INFO:teuthology.orchestra.run.trial026.stdout: Package Arch Version Repo Size 2026-02-22T21:35:37.498 INFO:teuthology.orchestra.run.trial026.stdout:================================================================================ 2026-02-22T21:35:37.498 INFO:teuthology.orchestra.run.trial026.stdout:Installing: 2026-02-22T21:35:37.498 INFO:teuthology.orchestra.run.trial026.stdout: librados-devel x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 126 k 2026-02-22T21:35:37.498 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:35:37.498 INFO:teuthology.orchestra.run.trial026.stdout:Transaction Summary 2026-02-22T21:35:37.498 INFO:teuthology.orchestra.run.trial026.stdout:================================================================================ 2026-02-22T21:35:37.498 INFO:teuthology.orchestra.run.trial026.stdout:Install 1 Package 2026-02-22T21:35:37.498 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:35:37.499 INFO:teuthology.orchestra.run.trial026.stdout:Total download size: 126 k 2026-02-22T21:35:37.499 INFO:teuthology.orchestra.run.trial026.stdout:Installed size: 456 k 2026-02-22T21:35:37.499 INFO:teuthology.orchestra.run.trial026.stdout:Downloading Packages: 2026-02-22T21:35:37.585 INFO:teuthology.orchestra.run.trial026.stdout:librados-devel-19.2.3-689.gf90f4eae.el9.x86_64. 1.4 MB/s | 126 kB 00:00 2026-02-22T21:35:37.585 INFO:teuthology.orchestra.run.trial026.stdout:-------------------------------------------------------------------------------- 2026-02-22T21:35:37.585 INFO:teuthology.orchestra.run.trial026.stdout:Total 1.4 MB/s | 126 kB 00:00 2026-02-22T21:35:37.586 INFO:teuthology.orchestra.run.trial026.stdout:Running transaction check 2026-02-22T21:35:37.588 INFO:teuthology.orchestra.run.trial026.stdout:Transaction check succeeded. 2026-02-22T21:35:37.588 INFO:teuthology.orchestra.run.trial026.stdout:Running transaction test 2026-02-22T21:35:37.621 INFO:teuthology.orchestra.run.trial026.stdout:Transaction test succeeded. 2026-02-22T21:35:37.621 INFO:teuthology.orchestra.run.trial026.stdout:Running transaction 2026-02-22T21:35:37.676 INFO:teuthology.orchestra.run.trial026.stdout: Preparing : 1/1 2026-02-22T21:35:37.693 INFO:teuthology.orchestra.run.trial026.stdout: Installing : librados-devel-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T21:35:37.775 INFO:teuthology.orchestra.run.trial026.stdout: Running scriptlet: librados-devel-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T21:35:37.834 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : librados-devel-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T21:35:37.834 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:35:37.834 INFO:teuthology.orchestra.run.trial026.stdout:Installed: 2026-02-22T21:35:37.834 INFO:teuthology.orchestra.run.trial026.stdout: librados-devel-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:35:37.834 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:35:37.834 INFO:teuthology.orchestra.run.trial026.stdout:Complete! 2026-02-22T21:35:37.865 DEBUG:teuthology.orchestra.run.trial026:> sudo yum -y install libcephfs2 2026-02-22T21:35:38.173 INFO:teuthology.orchestra.run.trial026.stdout:Last metadata expiration check: 0:00:39 ago on Sun 22 Feb 2026 09:34:59 PM UTC. 2026-02-22T21:35:38.237 INFO:teuthology.orchestra.run.trial026.stdout:Package libcephfs2-2:19.2.3-689.gf90f4eae.el9.x86_64 is already installed. 2026-02-22T21:35:38.255 INFO:teuthology.orchestra.run.trial026.stdout:Dependencies resolved. 2026-02-22T21:35:38.256 INFO:teuthology.orchestra.run.trial026.stdout:Nothing to do. 2026-02-22T21:35:38.256 INFO:teuthology.orchestra.run.trial026.stdout:Complete! 2026-02-22T21:35:38.277 DEBUG:teuthology.orchestra.run.trial026:> sudo yum -y install libcephfs-devel 2026-02-22T21:35:38.585 INFO:teuthology.orchestra.run.trial026.stdout:Last metadata expiration check: 0:00:39 ago on Sun 22 Feb 2026 09:34:59 PM UTC. 2026-02-22T21:35:38.670 INFO:teuthology.orchestra.run.trial026.stdout:Dependencies resolved. 2026-02-22T21:35:38.670 INFO:teuthology.orchestra.run.trial026.stdout:================================================================================ 2026-02-22T21:35:38.670 INFO:teuthology.orchestra.run.trial026.stdout: Package Arch Version Repo Size 2026-02-22T21:35:38.670 INFO:teuthology.orchestra.run.trial026.stdout:================================================================================ 2026-02-22T21:35:38.670 INFO:teuthology.orchestra.run.trial026.stdout:Installing: 2026-02-22T21:35:38.670 INFO:teuthology.orchestra.run.trial026.stdout: libcephfs-devel x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 34 k 2026-02-22T21:35:38.670 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:35:38.670 INFO:teuthology.orchestra.run.trial026.stdout:Transaction Summary 2026-02-22T21:35:38.670 INFO:teuthology.orchestra.run.trial026.stdout:================================================================================ 2026-02-22T21:35:38.670 INFO:teuthology.orchestra.run.trial026.stdout:Install 1 Package 2026-02-22T21:35:38.670 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:35:38.670 INFO:teuthology.orchestra.run.trial026.stdout:Total download size: 34 k 2026-02-22T21:35:38.670 INFO:teuthology.orchestra.run.trial026.stdout:Installed size: 153 k 2026-02-22T21:35:38.670 INFO:teuthology.orchestra.run.trial026.stdout:Downloading Packages: 2026-02-22T21:35:38.735 INFO:teuthology.orchestra.run.trial026.stdout:libcephfs-devel-19.2.3-689.gf90f4eae.el9.x86_64 527 kB/s | 34 kB 00:00 2026-02-22T21:35:38.735 INFO:teuthology.orchestra.run.trial026.stdout:-------------------------------------------------------------------------------- 2026-02-22T21:35:38.735 INFO:teuthology.orchestra.run.trial026.stdout:Total 519 kB/s | 34 kB 00:00 2026-02-22T21:35:38.736 INFO:teuthology.orchestra.run.trial026.stdout:Running transaction check 2026-02-22T21:35:38.737 INFO:teuthology.orchestra.run.trial026.stdout:Transaction check succeeded. 2026-02-22T21:35:38.737 INFO:teuthology.orchestra.run.trial026.stdout:Running transaction test 2026-02-22T21:35:38.740 INFO:teuthology.orchestra.run.trial026.stdout:Transaction test succeeded. 2026-02-22T21:35:38.740 INFO:teuthology.orchestra.run.trial026.stdout:Running transaction 2026-02-22T21:35:38.763 INFO:teuthology.orchestra.run.trial026.stdout: Preparing : 1/1 2026-02-22T21:35:38.779 INFO:teuthology.orchestra.run.trial026.stdout: Installing : libcephfs-devel-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T21:35:38.834 INFO:teuthology.orchestra.run.trial026.stdout: Running scriptlet: libcephfs-devel-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T21:35:38.891 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : libcephfs-devel-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T21:35:38.891 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:35:38.891 INFO:teuthology.orchestra.run.trial026.stdout:Installed: 2026-02-22T21:35:38.892 INFO:teuthology.orchestra.run.trial026.stdout: libcephfs-devel-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:35:38.892 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:35:38.892 INFO:teuthology.orchestra.run.trial026.stdout:Complete! 2026-02-22T21:35:38.918 DEBUG:teuthology.orchestra.run.trial026:> sudo yum -y install librados2 2026-02-22T21:35:39.229 INFO:teuthology.orchestra.run.trial026.stdout:Last metadata expiration check: 0:00:40 ago on Sun 22 Feb 2026 09:34:59 PM UTC. 2026-02-22T21:35:39.296 INFO:teuthology.orchestra.run.trial026.stdout:Package librados2-2:19.2.3-689.gf90f4eae.el9.x86_64 is already installed. 2026-02-22T21:35:39.315 INFO:teuthology.orchestra.run.trial026.stdout:Dependencies resolved. 2026-02-22T21:35:39.315 INFO:teuthology.orchestra.run.trial026.stdout:Nothing to do. 2026-02-22T21:35:39.316 INFO:teuthology.orchestra.run.trial026.stdout:Complete! 2026-02-22T21:35:39.338 DEBUG:teuthology.orchestra.run.trial026:> sudo yum -y install librbd1 2026-02-22T21:35:39.654 INFO:teuthology.orchestra.run.trial026.stdout:Last metadata expiration check: 0:00:40 ago on Sun 22 Feb 2026 09:34:59 PM UTC. 2026-02-22T21:35:39.721 INFO:teuthology.orchestra.run.trial026.stdout:Package librbd1-2:19.2.3-689.gf90f4eae.el9.x86_64 is already installed. 2026-02-22T21:35:39.740 INFO:teuthology.orchestra.run.trial026.stdout:Dependencies resolved. 2026-02-22T21:35:39.741 INFO:teuthology.orchestra.run.trial026.stdout:Nothing to do. 2026-02-22T21:35:39.741 INFO:teuthology.orchestra.run.trial026.stdout:Complete! 2026-02-22T21:35:39.762 DEBUG:teuthology.orchestra.run.trial026:> sudo yum -y install python3-rados 2026-02-22T21:35:40.076 INFO:teuthology.orchestra.run.trial026.stdout:Last metadata expiration check: 0:00:41 ago on Sun 22 Feb 2026 09:34:59 PM UTC. 2026-02-22T21:35:40.141 INFO:teuthology.orchestra.run.trial026.stdout:Package python3-rados-2:19.2.3-689.gf90f4eae.el9.x86_64 is already installed. 2026-02-22T21:35:40.159 INFO:teuthology.orchestra.run.trial026.stdout:Dependencies resolved. 2026-02-22T21:35:40.160 INFO:teuthology.orchestra.run.trial026.stdout:Nothing to do. 2026-02-22T21:35:40.160 INFO:teuthology.orchestra.run.trial026.stdout:Complete! 2026-02-22T21:35:40.183 DEBUG:teuthology.orchestra.run.trial026:> sudo yum -y install python3-rgw 2026-02-22T21:35:40.498 INFO:teuthology.orchestra.run.trial026.stdout:Last metadata expiration check: 0:00:41 ago on Sun 22 Feb 2026 09:34:59 PM UTC. 2026-02-22T21:35:40.563 INFO:teuthology.orchestra.run.trial026.stdout:Package python3-rgw-2:19.2.3-689.gf90f4eae.el9.x86_64 is already installed. 2026-02-22T21:35:40.581 INFO:teuthology.orchestra.run.trial026.stdout:Dependencies resolved. 2026-02-22T21:35:40.581 INFO:teuthology.orchestra.run.trial026.stdout:Nothing to do. 2026-02-22T21:35:40.582 INFO:teuthology.orchestra.run.trial026.stdout:Complete! 2026-02-22T21:35:40.603 DEBUG:teuthology.orchestra.run.trial026:> sudo yum -y install python3-cephfs 2026-02-22T21:35:40.917 INFO:teuthology.orchestra.run.trial026.stdout:Last metadata expiration check: 0:00:41 ago on Sun 22 Feb 2026 09:34:59 PM UTC. 2026-02-22T21:35:40.983 INFO:teuthology.orchestra.run.trial026.stdout:Package python3-cephfs-2:19.2.3-689.gf90f4eae.el9.x86_64 is already installed. 2026-02-22T21:35:41.001 INFO:teuthology.orchestra.run.trial026.stdout:Dependencies resolved. 2026-02-22T21:35:41.001 INFO:teuthology.orchestra.run.trial026.stdout:Nothing to do. 2026-02-22T21:35:41.001 INFO:teuthology.orchestra.run.trial026.stdout:Complete! 2026-02-22T21:35:41.022 DEBUG:teuthology.orchestra.run.trial026:> sudo yum -y install python3-rbd 2026-02-22T21:35:41.335 INFO:teuthology.orchestra.run.trial026.stdout:Last metadata expiration check: 0:00:42 ago on Sun 22 Feb 2026 09:34:59 PM UTC. 2026-02-22T21:35:41.401 INFO:teuthology.orchestra.run.trial026.stdout:Package python3-rbd-2:19.2.3-689.gf90f4eae.el9.x86_64 is already installed. 2026-02-22T21:35:41.420 INFO:teuthology.orchestra.run.trial026.stdout:Dependencies resolved. 2026-02-22T21:35:41.420 INFO:teuthology.orchestra.run.trial026.stdout:Nothing to do. 2026-02-22T21:35:41.420 INFO:teuthology.orchestra.run.trial026.stdout:Complete! 2026-02-22T21:35:41.442 DEBUG:teuthology.orchestra.run.trial026:> sudo yum -y install rbd-fuse 2026-02-22T21:35:41.757 INFO:teuthology.orchestra.run.trial026.stdout:Last metadata expiration check: 0:00:42 ago on Sun 22 Feb 2026 09:34:59 PM UTC. 2026-02-22T21:35:41.841 INFO:teuthology.orchestra.run.trial026.stdout:Dependencies resolved. 2026-02-22T21:35:41.842 INFO:teuthology.orchestra.run.trial026.stdout:================================================================================ 2026-02-22T21:35:41.842 INFO:teuthology.orchestra.run.trial026.stdout: Package Architecture Version Repository Size 2026-02-22T21:35:41.842 INFO:teuthology.orchestra.run.trial026.stdout:================================================================================ 2026-02-22T21:35:41.842 INFO:teuthology.orchestra.run.trial026.stdout:Installing: 2026-02-22T21:35:41.842 INFO:teuthology.orchestra.run.trial026.stdout: rbd-fuse x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 85 k 2026-02-22T21:35:41.842 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:35:41.842 INFO:teuthology.orchestra.run.trial026.stdout:Transaction Summary 2026-02-22T21:35:41.842 INFO:teuthology.orchestra.run.trial026.stdout:================================================================================ 2026-02-22T21:35:41.842 INFO:teuthology.orchestra.run.trial026.stdout:Install 1 Package 2026-02-22T21:35:41.842 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:35:41.842 INFO:teuthology.orchestra.run.trial026.stdout:Total download size: 85 k 2026-02-22T21:35:41.842 INFO:teuthology.orchestra.run.trial026.stdout:Installed size: 227 k 2026-02-22T21:35:41.842 INFO:teuthology.orchestra.run.trial026.stdout:Downloading Packages: 2026-02-22T21:35:41.918 INFO:teuthology.orchestra.run.trial026.stdout:rbd-fuse-19.2.3-689.gf90f4eae.el9.x86_64.rpm 1.1 MB/s | 85 kB 00:00 2026-02-22T21:35:41.918 INFO:teuthology.orchestra.run.trial026.stdout:-------------------------------------------------------------------------------- 2026-02-22T21:35:41.919 INFO:teuthology.orchestra.run.trial026.stdout:Total 1.1 MB/s | 85 kB 00:00 2026-02-22T21:35:41.919 INFO:teuthology.orchestra.run.trial026.stdout:Running transaction check 2026-02-22T21:35:41.922 INFO:teuthology.orchestra.run.trial026.stdout:Transaction check succeeded. 2026-02-22T21:35:41.922 INFO:teuthology.orchestra.run.trial026.stdout:Running transaction test 2026-02-22T21:35:41.954 INFO:teuthology.orchestra.run.trial026.stdout:Transaction test succeeded. 2026-02-22T21:35:41.955 INFO:teuthology.orchestra.run.trial026.stdout:Running transaction 2026-02-22T21:35:42.006 INFO:teuthology.orchestra.run.trial026.stdout: Preparing : 1/1 2026-02-22T21:35:42.023 INFO:teuthology.orchestra.run.trial026.stdout: Installing : rbd-fuse-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T21:35:42.107 INFO:teuthology.orchestra.run.trial026.stdout: Running scriptlet: rbd-fuse-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T21:35:42.165 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : rbd-fuse-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T21:35:42.166 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:35:42.166 INFO:teuthology.orchestra.run.trial026.stdout:Installed: 2026-02-22T21:35:42.166 INFO:teuthology.orchestra.run.trial026.stdout: rbd-fuse-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:35:42.166 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:35:42.166 INFO:teuthology.orchestra.run.trial026.stdout:Complete! 2026-02-22T21:35:42.194 DEBUG:teuthology.orchestra.run.trial026:> sudo yum -y install rbd-mirror 2026-02-22T21:35:42.516 INFO:teuthology.orchestra.run.trial026.stdout:Last metadata expiration check: 0:00:43 ago on Sun 22 Feb 2026 09:34:59 PM UTC. 2026-02-22T21:35:42.600 INFO:teuthology.orchestra.run.trial026.stdout:Dependencies resolved. 2026-02-22T21:35:42.600 INFO:teuthology.orchestra.run.trial026.stdout:================================================================================ 2026-02-22T21:35:42.600 INFO:teuthology.orchestra.run.trial026.stdout: Package Arch Version Repo Size 2026-02-22T21:35:42.600 INFO:teuthology.orchestra.run.trial026.stdout:================================================================================ 2026-02-22T21:35:42.600 INFO:teuthology.orchestra.run.trial026.stdout:Installing: 2026-02-22T21:35:42.600 INFO:teuthology.orchestra.run.trial026.stdout: rbd-mirror x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 3.1 M 2026-02-22T21:35:42.600 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:35:42.600 INFO:teuthology.orchestra.run.trial026.stdout:Transaction Summary 2026-02-22T21:35:42.600 INFO:teuthology.orchestra.run.trial026.stdout:================================================================================ 2026-02-22T21:35:42.600 INFO:teuthology.orchestra.run.trial026.stdout:Install 1 Package 2026-02-22T21:35:42.600 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:35:42.600 INFO:teuthology.orchestra.run.trial026.stdout:Total download size: 3.1 M 2026-02-22T21:35:42.600 INFO:teuthology.orchestra.run.trial026.stdout:Installed size: 13 M 2026-02-22T21:35:42.600 INFO:teuthology.orchestra.run.trial026.stdout:Downloading Packages: 2026-02-22T21:35:42.751 INFO:teuthology.orchestra.run.trial026.stdout:rbd-mirror-19.2.3-689.gf90f4eae.el9.x86_64.rpm 21 MB/s | 3.1 MB 00:00 2026-02-22T21:35:42.752 INFO:teuthology.orchestra.run.trial026.stdout:-------------------------------------------------------------------------------- 2026-02-22T21:35:42.752 INFO:teuthology.orchestra.run.trial026.stdout:Total 21 MB/s | 3.1 MB 00:00 2026-02-22T21:35:42.752 INFO:teuthology.orchestra.run.trial026.stdout:Running transaction check 2026-02-22T21:35:42.756 INFO:teuthology.orchestra.run.trial026.stdout:Transaction check succeeded. 2026-02-22T21:35:42.756 INFO:teuthology.orchestra.run.trial026.stdout:Running transaction test 2026-02-22T21:35:42.797 INFO:teuthology.orchestra.run.trial026.stdout:Transaction test succeeded. 2026-02-22T21:35:42.797 INFO:teuthology.orchestra.run.trial026.stdout:Running transaction 2026-02-22T21:35:42.989 INFO:teuthology.orchestra.run.trial026.stdout: Preparing : 1/1 2026-02-22T21:35:42.990 INFO:teuthology.orchestra.run.trial026.stdout: Installing : rbd-mirror-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T21:35:43.013 INFO:teuthology.orchestra.run.trial026.stdout: Running scriptlet: rbd-mirror-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T21:35:43.013 INFO:teuthology.orchestra.run.trial026.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-22T21:35:43.013 INFO:teuthology.orchestra.run.trial026.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2026-02-22T21:35:43.013 INFO:teuthology.orchestra.run.trial026.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2026-02-22T21:35:43.014 INFO:teuthology.orchestra.run.trial026.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2026-02-22T21:35:43.014 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:35:43.318 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : rbd-mirror-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T21:35:43.318 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:35:43.318 INFO:teuthology.orchestra.run.trial026.stdout:Installed: 2026-02-22T21:35:43.318 INFO:teuthology.orchestra.run.trial026.stdout: rbd-mirror-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:35:43.319 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:35:43.319 INFO:teuthology.orchestra.run.trial026.stdout:Complete! 2026-02-22T21:35:43.347 DEBUG:teuthology.orchestra.run.trial026:> sudo yum -y install rbd-nbd 2026-02-22T21:35:43.666 INFO:teuthology.orchestra.run.trial026.stdout:Last metadata expiration check: 0:00:44 ago on Sun 22 Feb 2026 09:34:59 PM UTC. 2026-02-22T21:35:43.751 INFO:teuthology.orchestra.run.trial026.stdout:Dependencies resolved. 2026-02-22T21:35:43.751 INFO:teuthology.orchestra.run.trial026.stdout:================================================================================ 2026-02-22T21:35:43.751 INFO:teuthology.orchestra.run.trial026.stdout: Package Architecture Version Repository Size 2026-02-22T21:35:43.751 INFO:teuthology.orchestra.run.trial026.stdout:================================================================================ 2026-02-22T21:35:43.751 INFO:teuthology.orchestra.run.trial026.stdout:Installing: 2026-02-22T21:35:43.751 INFO:teuthology.orchestra.run.trial026.stdout: rbd-nbd x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 170 k 2026-02-22T21:35:43.752 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:35:43.752 INFO:teuthology.orchestra.run.trial026.stdout:Transaction Summary 2026-02-22T21:35:43.752 INFO:teuthology.orchestra.run.trial026.stdout:================================================================================ 2026-02-22T21:35:43.752 INFO:teuthology.orchestra.run.trial026.stdout:Install 1 Package 2026-02-22T21:35:43.752 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:35:43.752 INFO:teuthology.orchestra.run.trial026.stdout:Total download size: 170 k 2026-02-22T21:35:43.752 INFO:teuthology.orchestra.run.trial026.stdout:Installed size: 490 k 2026-02-22T21:35:43.752 INFO:teuthology.orchestra.run.trial026.stdout:Downloading Packages: 2026-02-22T21:35:43.837 INFO:teuthology.orchestra.run.trial026.stdout:rbd-nbd-19.2.3-689.gf90f4eae.el9.x86_64.rpm 1.9 MB/s | 170 kB 00:00 2026-02-22T21:35:43.838 INFO:teuthology.orchestra.run.trial026.stdout:-------------------------------------------------------------------------------- 2026-02-22T21:35:43.838 INFO:teuthology.orchestra.run.trial026.stdout:Total 1.9 MB/s | 170 kB 00:00 2026-02-22T21:35:43.838 INFO:teuthology.orchestra.run.trial026.stdout:Running transaction check 2026-02-22T21:35:43.841 INFO:teuthology.orchestra.run.trial026.stdout:Transaction check succeeded. 2026-02-22T21:35:43.841 INFO:teuthology.orchestra.run.trial026.stdout:Running transaction test 2026-02-22T21:35:43.872 INFO:teuthology.orchestra.run.trial026.stdout:Transaction test succeeded. 2026-02-22T21:35:43.872 INFO:teuthology.orchestra.run.trial026.stdout:Running transaction 2026-02-22T21:35:43.928 INFO:teuthology.orchestra.run.trial026.stdout: Preparing : 1/1 2026-02-22T21:35:43.945 INFO:teuthology.orchestra.run.trial026.stdout: Installing : rbd-nbd-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T21:35:44.022 INFO:teuthology.orchestra.run.trial026.stdout: Running scriptlet: rbd-nbd-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T21:35:44.080 INFO:teuthology.orchestra.run.trial026.stdout: Verifying : rbd-nbd-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T21:35:44.081 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:35:44.081 INFO:teuthology.orchestra.run.trial026.stdout:Installed: 2026-02-22T21:35:44.081 INFO:teuthology.orchestra.run.trial026.stdout: rbd-nbd-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:35:44.081 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:35:44.081 INFO:teuthology.orchestra.run.trial026.stdout:Complete! 2026-02-22T21:35:44.108 DEBUG:teuthology.orchestra.run.trial026:> sudo yum -y install rbd-mirror 2026-02-22T21:35:44.438 INFO:teuthology.orchestra.run.trial026.stdout:Last metadata expiration check: 0:00:45 ago on Sun 22 Feb 2026 09:34:59 PM UTC. 2026-02-22T21:35:44.504 INFO:teuthology.orchestra.run.trial026.stdout:Package rbd-mirror-2:19.2.3-689.gf90f4eae.el9.x86_64 is already installed. 2026-02-22T21:35:44.522 INFO:teuthology.orchestra.run.trial026.stdout:Dependencies resolved. 2026-02-22T21:35:44.523 INFO:teuthology.orchestra.run.trial026.stdout:Nothing to do. 2026-02-22T21:35:44.523 INFO:teuthology.orchestra.run.trial026.stdout:Complete! 2026-02-22T21:35:44.547 DEBUG:teuthology.parallel:result is None 2026-02-22T21:35:44.548 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=f90f4eaebb28b619cf76e80cbbe997656da69579 2026-02-22T21:35:44.696 DEBUG:teuthology.orchestra.run.trial004:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2026-02-22T21:35:44.717 INFO:teuthology.orchestra.run.trial004.stdout:19.2.3-689.gf90f4eae.el9 2026-02-22T21:35:44.718 INFO:teuthology.packaging:The installed version of ceph is 19.2.3-689.gf90f4eae.el9 2026-02-22T21:35:44.718 INFO:teuthology.task.install:The correct ceph version 19.2.3-689.gf90f4eae is installed. 2026-02-22T21:35:44.721 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=f90f4eaebb28b619cf76e80cbbe997656da69579 2026-02-22T21:35:44.854 DEBUG:teuthology.orchestra.run.trial026:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2026-02-22T21:35:44.874 INFO:teuthology.orchestra.run.trial026.stdout:19.2.3-689.gf90f4eae.el9 2026-02-22T21:35:44.875 INFO:teuthology.packaging:The installed version of ceph is 19.2.3-689.gf90f4eae.el9 2026-02-22T21:35:44.875 INFO:teuthology.task.install:The correct ceph version 19.2.3-689.gf90f4eae is installed. 2026-02-22T21:35:44.877 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2026-02-22T21:35:44.877 DEBUG:teuthology.orchestra.run.trial004:> set -ex 2026-02-22T21:35:44.877 DEBUG:teuthology.orchestra.run.trial004:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2026-02-22T21:35:44.904 DEBUG:teuthology.orchestra.run.trial026:> set -ex 2026-02-22T21:35:44.904 DEBUG:teuthology.orchestra.run.trial026:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2026-02-22T21:35:44.939 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2026-02-22T21:35:44.940 DEBUG:teuthology.orchestra.run.trial004:> set -ex 2026-02-22T21:35:44.940 DEBUG:teuthology.orchestra.run.trial004:> sudo dd of=/usr/bin/daemon-helper 2026-02-22T21:35:44.968 DEBUG:teuthology.orchestra.run.trial004:> sudo chmod a=rx -- /usr/bin/daemon-helper 2026-02-22T21:35:45.032 DEBUG:teuthology.orchestra.run.trial026:> set -ex 2026-02-22T21:35:45.032 DEBUG:teuthology.orchestra.run.trial026:> sudo dd of=/usr/bin/daemon-helper 2026-02-22T21:35:45.057 DEBUG:teuthology.orchestra.run.trial026:> sudo chmod a=rx -- /usr/bin/daemon-helper 2026-02-22T21:35:45.120 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2026-02-22T21:35:45.120 DEBUG:teuthology.orchestra.run.trial004:> set -ex 2026-02-22T21:35:45.120 DEBUG:teuthology.orchestra.run.trial004:> sudo dd of=/usr/bin/adjust-ulimits 2026-02-22T21:35:45.145 DEBUG:teuthology.orchestra.run.trial004:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2026-02-22T21:35:45.208 DEBUG:teuthology.orchestra.run.trial026:> set -ex 2026-02-22T21:35:45.208 DEBUG:teuthology.orchestra.run.trial026:> sudo dd of=/usr/bin/adjust-ulimits 2026-02-22T21:35:45.233 DEBUG:teuthology.orchestra.run.trial026:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2026-02-22T21:35:45.299 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2026-02-22T21:35:45.299 DEBUG:teuthology.orchestra.run.trial004:> set -ex 2026-02-22T21:35:45.300 DEBUG:teuthology.orchestra.run.trial004:> sudo dd of=/usr/bin/stdin-killer 2026-02-22T21:35:45.325 DEBUG:teuthology.orchestra.run.trial004:> sudo chmod a=rx -- /usr/bin/stdin-killer 2026-02-22T21:35:45.389 DEBUG:teuthology.orchestra.run.trial026:> set -ex 2026-02-22T21:35:45.390 DEBUG:teuthology.orchestra.run.trial026:> sudo dd of=/usr/bin/stdin-killer 2026-02-22T21:35:45.414 DEBUG:teuthology.orchestra.run.trial026:> sudo chmod a=rx -- /usr/bin/stdin-killer 2026-02-22T21:35:45.479 INFO:teuthology.run_tasks:Running task ceph... 2026-02-22T21:35:45.591 INFO:tasks.ceph:Making ceph log dir writeable by non-root... 2026-02-22T21:35:45.591 DEBUG:teuthology.orchestra.run.trial004:> sudo chmod 777 /var/log/ceph 2026-02-22T21:35:45.593 DEBUG:teuthology.orchestra.run.trial026:> sudo chmod 777 /var/log/ceph 2026-02-22T21:35:45.617 INFO:tasks.ceph:Disabling ceph logrotate... 2026-02-22T21:35:45.617 DEBUG:teuthology.orchestra.run.trial004:> sudo rm -f -- /etc/logrotate.d/ceph 2026-02-22T21:35:45.657 DEBUG:teuthology.orchestra.run.trial026:> sudo rm -f -- /etc/logrotate.d/ceph 2026-02-22T21:35:45.681 INFO:tasks.ceph:Creating extra log directories... 2026-02-22T21:35:45.681 DEBUG:teuthology.orchestra.run.trial004:> sudo install -d -m0777 -- /var/log/ceph/valgrind /var/log/ceph/profiling-logger 2026-02-22T21:35:45.719 DEBUG:teuthology.orchestra.run.trial026:> sudo install -d -m0777 -- /var/log/ceph/valgrind /var/log/ceph/profiling-logger 2026-02-22T21:35:45.754 INFO:tasks.ceph:Creating ceph cluster cluster1... 2026-02-22T21:35:45.754 INFO:tasks.ceph:config {'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'simple'}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bluestore block size': 96636764160, 'bluestore compression algorithm': 'lz4', 'bluestore compression mode': 'aggressive', 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'fs': 'xfs', 'mkfs_options': None, 'mount_options': None, 'skip_mgr_daemons': False, 'log_ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'cpu_profile': set(), 'cluster': 'cluster1', 'mon_bind_msgr2': True, 'mon_bind_addrvec': True} 2026-02-22T21:35:45.754 INFO:tasks.ceph:ctx.config {'archive_path': '/home/teuthworker/mnt/teuthology/yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial/64060', 'branch': 'wip-yuri6-testing-2026-02-22-1629-squid', 'description': 'rbd/mirror-thrash/{base/install clients/mirror cluster/{2-node openstack} conf/{disable-pool-app} msgr-failures/few objectstore/bluestore-comp-lz4 policy/simple rbd-mirror/four-per-cluster supported-random-distro$/{centos_latest} workloads/rbd-mirror-snapshot-stress-workunit-exclusive-lock}', 'email': 'yweinste@redhat.com', 'first_in_suite': False, 'flavor': 'default', 'job_id': '64060', 'kernel': {'branch': 'distro', 'kdb': True, 'sha1': 'distro'}, 'last_in_suite': False, 'machine_type': 'trial', 'meta': [{'desc': 'run two ceph clusters and install rbd-mirror'}, {'desc': 'configure the permissions for client.mirror'}, {'desc': '2 ceph clusters with 1 mon and 3 osds each'}, {'desc': 'run four rbd-mirror daemons per cluster'}, {'desc': 'run the rbd_mirror_stress.sh workunit to test the rbd-mirror daemon'}], 'name': 'yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial', 'no_nested_subset': False, 'openstack': [{'volumes': {'count': 3, 'size': 30}}], 'os_type': 'centos', 'os_version': '9.stream', 'overrides': {'admin_socket': {'branch': 'wip-yuri6-testing-2026-02-22-1629-squid'}, 'ceph': {'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'simple'}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bluestore block size': 96636764160, 'bluestore compression algorithm': 'lz4', 'bluestore compression mode': 'aggressive', 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579'}, 'ceph-deploy': {'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {}}}, 'install': {'ceph': {'flavor': 'default', 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579'}}, 'selinux': {'allowlist': ['scontext=system_u:system_r:getty_t:s0']}, 'thrashosds': {'bdev_inject_crash': 2, 'bdev_inject_crash_probability': 0.5}, 'workunit': {'branch': 'wip-yuri6-testing-2026-02-22-1629-squid', 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579'}}, 'owner': 'scheduled_yuriw@soko04.front.sepia.ceph.com', 'priority': 69, 'repo': 'https://github.com/ceph/ceph-ci.git', 'roles': [['cluster1.mon.a', 'cluster1.mgr.x', 'cluster2.mgr.x', 'cluster1.osd.0', 'cluster1.osd.1', 'cluster1.osd.2', 'cluster1.client.0', 'cluster2.client.0'], ['cluster2.mon.a', 'cluster2.osd.0', 'cluster2.osd.1', 'cluster2.osd.2', 'cluster1.client.mirror', 'cluster1.client.mirror.0', 'cluster1.client.mirror.1', 'cluster1.client.mirror.2', 'cluster1.client.mirror.3', 'cluster1.client.mirror.4', 'cluster1.client.mirror.5', 'cluster1.client.mirror.6', 'cluster2.client.mirror', 'cluster2.client.mirror.0', 'cluster2.client.mirror.1', 'cluster2.client.mirror.2', 'cluster2.client.mirror.3', 'cluster2.client.mirror.4', 'cluster2.client.mirror.5', 'cluster2.client.mirror.6']], 'seed': 9196, 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579', 'sleep_before_teardown': 0, 'subset': '111/120000', 'suite': 'rbd', 'suite_branch': 'wip-yuri6-testing-2026-02-22-1629-squid', 'suite_path': '/home/teuthworker/src/github.com_ceph_ceph-c_f90f4eaebb28b619cf76e80cbbe997656da69579/qa', 'suite_relpath': 'qa', 'suite_repo': 'https://github.com/ceph/ceph-ci.git', 'suite_sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579', 'targets': {'trial004.front.sepia.ceph.com': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBDkPjQYprX4CF30+jAbTd9L3HjPvsP5GsEYCy0m1NpRHUc7PIKylXMo7Pk3bw2gAMNlmx9b6E/JJW/eTIjrHUGg=', 'trial026.front.sepia.ceph.com': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBM5AXTZqVQNBK8dSoTzQSzjoGT4p0yrUpjtrv9BGY3wKuaRfx3V9PHQ5dxCaBeyOqt+DwHWF93iRGCUbLiIW/wQ='}, 'tasks': [{'internal.check_packages': None}, {'internal.buildpackages_prep': None}, {'internal.save_config': None}, {'internal.check_lock': None}, {'internal.add_remotes': None}, {'console_log': None}, {'internal.connect': None}, {'internal.push_inventory': None}, {'internal.serialize_remote_roles': None}, {'internal.check_conflict': None}, {'internal.check_ceph_data': None}, {'internal.vm_setup': None}, {'kernel': {'branch': 'distro', 'kdb': True, 'sha1': 'distro'}}, {'internal.base': None}, {'internal.archive_upload': None}, {'internal.archive': None}, {'internal.coredump': None}, {'internal.sudo': None}, {'internal.syslog': None}, {'internal.timer': None}, {'pcp': None}, {'selinux': None}, {'ansible.cephlab': None}, {'clock': None}, {'install': {'extra_packages': ['rbd-mirror'], 'flavor': 'default', 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579'}}, {'ceph': {'cluster': 'cluster1', 'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'simple'}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bluestore block size': 96636764160, 'bluestore compression algorithm': 'lz4', 'bluestore compression mode': 'aggressive', 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579'}}, {'ceph': {'cluster': 'cluster2'}}, {'exec': {'cluster1.client.mirror.0': ["sudo ceph --cluster cluster1 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd'"]}}, {'rbd-mirror': {'client': 'cluster1.client.mirror.0', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster1.client.mirror.1', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster1.client.mirror.2', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster1.client.mirror.3', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster2.client.mirror.0', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster2.client.mirror.1', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster2.client.mirror.2', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster2.client.mirror.3', 'thrash': True}}, {'rbd-mirror-thrash': {'cluster': 'cluster1'}}, {'rbd-mirror-thrash': {'cluster': 'cluster2'}}, {'workunit': {'clients': {'cluster1.client.mirror': ['rbd/rbd_mirror_stress.sh']}, 'env': {'CEPH_ARGS': '', 'MIRROR_POOL_MODE': 'image', 'RBD_IMAGE_FEATURES': 'layering,exclusive-lock', 'RBD_MIRROR_INSTANCES': '4', 'RBD_MIRROR_MODE': 'snapshot', 'RBD_MIRROR_USE_EXISTING_CLUSTER': '1', 'RBD_MIRROR_USE_RBD_MIRROR': '1'}, 'timeout': '6h'}}], 'teuthology': {'fragments_dropped': [], 'meta': {}, 'postmerge': []}, 'teuthology_branch': 'main', 'teuthology_sha1': 'fa17720d0088c3ac28e473468bfc79eeaff5cd38', 'timestamp': '2026-02-22_19:35:38', 'tube': 'trial', 'user': 'yuriw', 'verbose': True, 'worker_log': '/home/teuthworker/mnt/teuthology/worker_logs/dispatcher.trial.3318653'} 2026-02-22T21:35:45.754 DEBUG:teuthology.orchestra.run.trial004:> install -d -m0755 -- /home/ubuntu/cephtest/cluster1.data 2026-02-22T21:35:45.783 DEBUG:teuthology.orchestra.run.trial026:> install -d -m0755 -- /home/ubuntu/cephtest/cluster1.data 2026-02-22T21:35:45.803 DEBUG:teuthology.orchestra.run.trial004:> sudo install -d -m0777 -- /var/run/ceph 2026-02-22T21:35:45.838 DEBUG:teuthology.orchestra.run.trial026:> sudo install -d -m0777 -- /var/run/ceph 2026-02-22T21:35:45.865 DEBUG:teuthology.orchestra.run.trial004:> set -ex 2026-02-22T21:35:45.865 DEBUG:teuthology.orchestra.run.trial004:> dd if=/scratch_devs of=/dev/stdout 2026-02-22T21:35:45.914 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2026-02-22T21:35:45.914 DEBUG:teuthology.orchestra.run.trial004:> stat /dev/vg_nvme/lv_1 2026-02-22T21:35:45.971 INFO:teuthology.orchestra.run.trial004.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2026-02-22T21:35:45.971 INFO:teuthology.orchestra.run.trial004.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-02-22T21:35:45.971 INFO:teuthology.orchestra.run.trial004.stdout:Device: 6h/6d Inode: 1298 Links: 1 2026-02-22T21:35:45.971 INFO:teuthology.orchestra.run.trial004.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-02-22T21:35:45.971 INFO:teuthology.orchestra.run.trial004.stdout:Context: system_u:object_r:device_t:s0 2026-02-22T21:35:45.972 INFO:teuthology.orchestra.run.trial004.stdout:Access: 2026-02-22 21:35:20.467371701 +0000 2026-02-22T21:35:45.972 INFO:teuthology.orchestra.run.trial004.stdout:Modify: 2026-02-22 21:34:42.356163711 +0000 2026-02-22T21:35:45.972 INFO:teuthology.orchestra.run.trial004.stdout:Change: 2026-02-22 21:34:42.356163711 +0000 2026-02-22T21:35:45.972 INFO:teuthology.orchestra.run.trial004.stdout: Birth: 2026-02-22 21:34:42.356163711 +0000 2026-02-22T21:35:45.972 DEBUG:teuthology.orchestra.run.trial004:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2026-02-22T21:35:46.033 INFO:teuthology.orchestra.run.trial004.stderr:1+0 records in 2026-02-22T21:35:46.033 INFO:teuthology.orchestra.run.trial004.stderr:1+0 records out 2026-02-22T21:35:46.034 INFO:teuthology.orchestra.run.trial004.stderr:512 bytes copied, 0.000152135 s, 3.4 MB/s 2026-02-22T21:35:46.034 DEBUG:teuthology.orchestra.run.trial004:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2026-02-22T21:35:46.091 DEBUG:teuthology.orchestra.run.trial004:> stat /dev/vg_nvme/lv_2 2026-02-22T21:35:46.147 INFO:teuthology.orchestra.run.trial004.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2026-02-22T21:35:46.148 INFO:teuthology.orchestra.run.trial004.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-02-22T21:35:46.148 INFO:teuthology.orchestra.run.trial004.stdout:Device: 6h/6d Inode: 1311 Links: 1 2026-02-22T21:35:46.148 INFO:teuthology.orchestra.run.trial004.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-02-22T21:35:46.148 INFO:teuthology.orchestra.run.trial004.stdout:Context: system_u:object_r:device_t:s0 2026-02-22T21:35:46.148 INFO:teuthology.orchestra.run.trial004.stdout:Access: 2026-02-22 21:35:20.467371701 +0000 2026-02-22T21:35:46.148 INFO:teuthology.orchestra.run.trial004.stdout:Modify: 2026-02-22 21:34:42.357163691 +0000 2026-02-22T21:35:46.148 INFO:teuthology.orchestra.run.trial004.stdout:Change: 2026-02-22 21:34:42.357163691 +0000 2026-02-22T21:35:46.148 INFO:teuthology.orchestra.run.trial004.stdout: Birth: 2026-02-22 21:34:42.357163691 +0000 2026-02-22T21:35:46.149 DEBUG:teuthology.orchestra.run.trial004:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2026-02-22T21:35:46.210 INFO:teuthology.orchestra.run.trial004.stderr:1+0 records in 2026-02-22T21:35:46.210 INFO:teuthology.orchestra.run.trial004.stderr:1+0 records out 2026-02-22T21:35:46.210 INFO:teuthology.orchestra.run.trial004.stderr:512 bytes copied, 0.000129753 s, 3.9 MB/s 2026-02-22T21:35:46.211 DEBUG:teuthology.orchestra.run.trial004:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2026-02-22T21:35:46.267 DEBUG:teuthology.orchestra.run.trial004:> stat /dev/vg_nvme/lv_3 2026-02-22T21:35:46.323 INFO:teuthology.orchestra.run.trial004.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2026-02-22T21:35:46.324 INFO:teuthology.orchestra.run.trial004.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-02-22T21:35:46.324 INFO:teuthology.orchestra.run.trial004.stdout:Device: 6h/6d Inode: 1299 Links: 1 2026-02-22T21:35:46.324 INFO:teuthology.orchestra.run.trial004.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-02-22T21:35:46.324 INFO:teuthology.orchestra.run.trial004.stdout:Context: system_u:object_r:device_t:s0 2026-02-22T21:35:46.324 INFO:teuthology.orchestra.run.trial004.stdout:Access: 2026-02-22 21:35:20.468371679 +0000 2026-02-22T21:35:46.324 INFO:teuthology.orchestra.run.trial004.stdout:Modify: 2026-02-22 21:34:42.356163711 +0000 2026-02-22T21:35:46.324 INFO:teuthology.orchestra.run.trial004.stdout:Change: 2026-02-22 21:34:42.356163711 +0000 2026-02-22T21:35:46.324 INFO:teuthology.orchestra.run.trial004.stdout: Birth: 2026-02-22 21:34:42.356163711 +0000 2026-02-22T21:35:46.325 DEBUG:teuthology.orchestra.run.trial004:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2026-02-22T21:35:46.386 INFO:teuthology.orchestra.run.trial004.stderr:1+0 records in 2026-02-22T21:35:46.387 INFO:teuthology.orchestra.run.trial004.stderr:1+0 records out 2026-02-22T21:35:46.387 INFO:teuthology.orchestra.run.trial004.stderr:512 bytes copied, 0.000149279 s, 3.4 MB/s 2026-02-22T21:35:46.388 DEBUG:teuthology.orchestra.run.trial004:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2026-02-22T21:35:46.445 DEBUG:teuthology.orchestra.run.trial004:> stat /dev/vg_nvme/lv_4 2026-02-22T21:35:46.501 INFO:teuthology.orchestra.run.trial004.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2026-02-22T21:35:46.501 INFO:teuthology.orchestra.run.trial004.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-02-22T21:35:46.501 INFO:teuthology.orchestra.run.trial004.stdout:Device: 6h/6d Inode: 1310 Links: 1 2026-02-22T21:35:46.502 INFO:teuthology.orchestra.run.trial004.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-02-22T21:35:46.502 INFO:teuthology.orchestra.run.trial004.stdout:Context: system_u:object_r:device_t:s0 2026-02-22T21:35:46.502 INFO:teuthology.orchestra.run.trial004.stdout:Access: 2026-02-22 21:35:20.468371679 +0000 2026-02-22T21:35:46.502 INFO:teuthology.orchestra.run.trial004.stdout:Modify: 2026-02-22 21:34:42.357163691 +0000 2026-02-22T21:35:46.502 INFO:teuthology.orchestra.run.trial004.stdout:Change: 2026-02-22 21:34:42.357163691 +0000 2026-02-22T21:35:46.502 INFO:teuthology.orchestra.run.trial004.stdout: Birth: 2026-02-22 21:34:42.357163691 +0000 2026-02-22T21:35:46.502 DEBUG:teuthology.orchestra.run.trial004:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2026-02-22T21:35:46.563 INFO:teuthology.orchestra.run.trial004.stderr:1+0 records in 2026-02-22T21:35:46.563 INFO:teuthology.orchestra.run.trial004.stderr:1+0 records out 2026-02-22T21:35:46.563 INFO:teuthology.orchestra.run.trial004.stderr:512 bytes copied, 0.000146154 s, 3.5 MB/s 2026-02-22T21:35:46.565 DEBUG:teuthology.orchestra.run.trial004:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2026-02-22T21:35:46.623 INFO:tasks.ceph:osd dev map: {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'} 2026-02-22T21:35:46.623 INFO:tasks.ceph:remote_to_roles_to_devs: {Remote(name='ubuntu@trial004.front.sepia.ceph.com'): {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'}} 2026-02-22T21:35:46.623 INFO:tasks.ceph:Generating config... 2026-02-22T21:35:46.623 INFO:tasks.ceph:[client] debug rbd = 20 2026-02-22T21:35:46.624 INFO:tasks.ceph:[client] debug rbd_mirror = 15 2026-02-22T21:35:46.624 INFO:tasks.ceph:[client] log to stderr = False 2026-02-22T21:35:46.624 INFO:tasks.ceph:[client] rbd default features = 125 2026-02-22T21:35:46.624 INFO:tasks.ceph:[client] rbd mirror image policy type = simple 2026-02-22T21:35:46.624 INFO:tasks.ceph:[client.mirror.0] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T21:35:46.624 INFO:tasks.ceph:[client.mirror.0] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T21:35:46.624 INFO:tasks.ceph:[client.mirror.1] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T21:35:46.624 INFO:tasks.ceph:[client.mirror.1] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T21:35:46.624 INFO:tasks.ceph:[client.mirror.2] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T21:35:46.624 INFO:tasks.ceph:[client.mirror.2] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T21:35:46.624 INFO:tasks.ceph:[client.mirror.3] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T21:35:46.624 INFO:tasks.ceph:[client.mirror.3] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T21:35:46.624 INFO:tasks.ceph:[global] mon client directed command retry = 5 2026-02-22T21:35:46.624 INFO:tasks.ceph:[global] mon warn on pool no app = False 2026-02-22T21:35:46.624 INFO:tasks.ceph:[global] ms inject socket failures = 5000 2026-02-22T21:35:46.624 INFO:tasks.ceph:[mgr] debug mgr = 20 2026-02-22T21:35:46.624 INFO:tasks.ceph:[mgr] debug ms = 1 2026-02-22T21:35:46.624 INFO:tasks.ceph:[mon] debug mon = 20 2026-02-22T21:35:46.624 INFO:tasks.ceph:[mon] debug ms = 1 2026-02-22T21:35:46.624 INFO:tasks.ceph:[mon] debug paxos = 20 2026-02-22T21:35:46.624 INFO:tasks.ceph:[osd] bluestore block size = 96636764160 2026-02-22T21:35:46.624 INFO:tasks.ceph:[osd] bluestore compression algorithm = lz4 2026-02-22T21:35:46.624 INFO:tasks.ceph:[osd] bluestore compression mode = aggressive 2026-02-22T21:35:46.624 INFO:tasks.ceph:[osd] bluestore fsck on mount = True 2026-02-22T21:35:46.624 INFO:tasks.ceph:[osd] debug bluefs = 1/20 2026-02-22T21:35:46.624 INFO:tasks.ceph:[osd] debug bluestore = 1/20 2026-02-22T21:35:46.624 INFO:tasks.ceph:[osd] debug ms = 1 2026-02-22T21:35:46.624 INFO:tasks.ceph:[osd] debug osd = 20 2026-02-22T21:35:46.624 INFO:tasks.ceph:[osd] debug rocksdb = 4/10 2026-02-22T21:35:46.624 INFO:tasks.ceph:[osd] mon osd backfillfull_ratio = 0.85 2026-02-22T21:35:46.624 INFO:tasks.ceph:[osd] mon osd full ratio = 0.9 2026-02-22T21:35:46.624 INFO:tasks.ceph:[osd] mon osd nearfull ratio = 0.8 2026-02-22T21:35:46.624 INFO:tasks.ceph:[osd] osd failsafe full ratio = 0.95 2026-02-22T21:35:46.624 INFO:tasks.ceph:[osd] osd objectstore = bluestore 2026-02-22T21:35:46.625 INFO:tasks.ceph:Setting up cluster1.mon.a... 2026-02-22T21:35:46.625 DEBUG:teuthology.orchestra.run.trial004:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring /etc/ceph/cluster1.keyring 2026-02-22T21:35:46.698 INFO:teuthology.orchestra.run.trial004.stdout:creating /etc/ceph/cluster1.keyring 2026-02-22T21:35:46.702 DEBUG:teuthology.orchestra.run.trial004:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=mon. /etc/ceph/cluster1.keyring 2026-02-22T21:35:46.778 DEBUG:teuthology.orchestra.run.trial004:> sudo chmod 0644 /etc/ceph/cluster1.keyring 2026-02-22T21:35:46.844 DEBUG:tasks.ceph:Ceph mon addresses: [('cluster1.mon.a', '10.20.193.4')] 2026-02-22T21:35:46.845 DEBUG:tasks.ceph:writing out conf {'global': {'chdir': '', 'pid file': '/var/run/ceph/$cluster-$name.pid', 'auth supported': 'cephx', 'filestore xattr use omap': 'true', 'mon clock drift allowed': '1.000', 'osd crush chooseleaf type': '0', 'auth debug': 'true', 'ms die on old message': 'true', 'ms die on bug': 'true', 'mon max pg per osd': '10000', 'mon pg warn max object skew': '0', 'osd_pool_default_pg_autoscale_mode': 'off', 'osd pool default size': '2', 'mon osd allow primary affinity': 'true', 'mon osd allow pg remap': 'true', 'mon warn on legacy crush tunables': 'false', 'mon warn on crush straw calc version zero': 'false', 'mon warn on no sortbitwise': 'false', 'mon warn on osd down out interval zero': 'false', 'mon warn on too few osds': 'false', 'mon_warn_on_pool_pg_num_not_power_of_two': 'false', 'mon_warn_on_pool_no_redundancy': 'false', 'mon_allow_pool_size_one': 'true', 'osd pool default erasure code profile': 'plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd', 'osd default data pool replay window': '5', 'mon allow pool delete': 'true', 'mon cluster log file level': 'debug', 'debug asserts on shutdown': 'true', 'mon health detail to clog': 'false', 'mon host': '10.20.193.4', 'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'osd': {'osd journal size': '100', 'osd scrub load threshold': '5.0', 'osd scrub max interval': '600', 'osd mclock profile': 'high_recovery_ops', 'osd mclock skip benchmark': 'true', 'osd recover clone overlap': 'true', 'osd recovery max chunk': '1048576', 'osd debug shutdown': 'true', 'osd debug op order': 'true', 'osd debug verify stray on activate': 'true', 'osd debug trim objects': 'true', 'osd open classes on start': 'true', 'osd debug pg log writeout': 'true', 'osd deep scrub update digest min age': '30', 'osd map max advance': '10', 'journal zero on create': 'true', 'filestore ondisk finisher threads': '3', 'filestore apply finisher threads': '3', 'bdev debug aio': 'true', 'osd debug misdirected ops': 'true', 'bluestore block size': 96636764160, 'bluestore compression algorithm': 'lz4', 'bluestore compression mode': 'aggressive', 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}, 'mgr': {'debug ms': 1, 'debug mgr': 20, 'debug mon': '20', 'debug auth': '20', 'mon reweight min pgs per osd': '4', 'mon reweight min bytes per osd': '10', 'mgr/telemetry/nag': 'false'}, 'mon': {'debug ms': 1, 'debug mon': 20, 'debug paxos': 20, 'debug auth': '20', 'mon data avail warn': '5', 'mon mgr mkfs grace': '240', 'mon reweight min pgs per osd': '4', 'mon osd reporter subtree level': 'osd', 'mon osd prime pg temp': 'true', 'mon reweight min bytes per osd': '10', 'auth mon ticket ttl': '660', 'auth service ticket ttl': '240', 'mon_warn_on_insecure_global_id_reclaim': 'false', 'mon_warn_on_insecure_global_id_reclaim_allowed': 'false', 'mon_down_mkfs_grace': '2m', 'mon_warn_on_filestore_osds': 'false'}, 'client': {'rgw cache enabled': 'true', 'rgw enable ops log': 'true', 'rgw enable usage log': 'true', 'log file': '/var/log/ceph/$cluster-$name.$pid.log', 'admin socket': '/var/run/ceph/$cluster-$name.$pid.asok', 'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'simple'}, 'mon.a': {}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}} 2026-02-22T21:35:46.847 DEBUG:teuthology.orchestra.run.trial004:> set -ex 2026-02-22T21:35:46.847 DEBUG:teuthology.orchestra.run.trial004:> dd of=/home/ubuntu/cephtest/ceph.tmp.conf 2026-02-22T21:35:46.901 DEBUG:teuthology.orchestra.run.trial004:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage monmaptool -c /home/ubuntu/cephtest/ceph.tmp.conf --create --clobber --enable-all-features --add a 10.20.193.4 --print /home/ubuntu/cephtest/cluster1.monmap 2026-02-22T21:35:46.972 INFO:teuthology.orchestra.run.trial004.stdout:monmaptool: monmap file /home/ubuntu/cephtest/cluster1.monmap 2026-02-22T21:35:46.973 INFO:teuthology.orchestra.run.trial004.stdout:monmaptool: generated fsid 54a0d3ac-2ee3-469c-a716-4416b2f462d9 2026-02-22T21:35:46.973 INFO:teuthology.orchestra.run.trial004.stdout:setting min_mon_release = quincy 2026-02-22T21:35:46.973 INFO:teuthology.orchestra.run.trial004.stdout:epoch 0 2026-02-22T21:35:46.973 INFO:teuthology.orchestra.run.trial004.stdout:fsid 54a0d3ac-2ee3-469c-a716-4416b2f462d9 2026-02-22T21:35:46.973 INFO:teuthology.orchestra.run.trial004.stdout:last_changed 2026-02-22T21:35:46.972084+0000 2026-02-22T21:35:46.973 INFO:teuthology.orchestra.run.trial004.stdout:created 2026-02-22T21:35:46.972084+0000 2026-02-22T21:35:46.973 INFO:teuthology.orchestra.run.trial004.stdout:min_mon_release 17 (quincy) 2026-02-22T21:35:46.973 INFO:teuthology.orchestra.run.trial004.stdout:election_strategy: 1 2026-02-22T21:35:46.973 INFO:teuthology.orchestra.run.trial004.stdout:0: [v2:10.20.193.4:3300/0,v1:10.20.193.4:6789/0] mon.a 2026-02-22T21:35:46.973 INFO:teuthology.orchestra.run.trial004.stdout:monmaptool: writing epoch 0 to /home/ubuntu/cephtest/cluster1.monmap (1 monitors) 2026-02-22T21:35:46.975 DEBUG:teuthology.orchestra.run.trial004:> rm -- /home/ubuntu/cephtest/ceph.tmp.conf 2026-02-22T21:35:47.032 INFO:tasks.ceph:Writing /etc/ceph/cluster1.conf for FSID 54a0d3ac-2ee3-469c-a716-4416b2f462d9... 2026-02-22T21:35:47.033 DEBUG:teuthology.orchestra.run.trial004:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/cluster1.conf && sudo chmod 0644 /etc/ceph/cluster1.conf > /dev/null 2026-02-22T21:35:47.074 DEBUG:teuthology.orchestra.run.trial026:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/cluster1.conf && sudo chmod 0644 /etc/ceph/cluster1.conf > /dev/null 2026-02-22T21:35:47.111 INFO:teuthology.orchestra.run.trial004.stdout:[global] 2026-02-22T21:35:47.111 INFO:teuthology.orchestra.run.trial004.stdout: chdir = "" 2026-02-22T21:35:47.111 INFO:teuthology.orchestra.run.trial004.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2026-02-22T21:35:47.111 INFO:teuthology.orchestra.run.trial004.stdout: auth supported = cephx 2026-02-22T21:35:47.111 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:35:47.111 INFO:teuthology.orchestra.run.trial004.stdout: filestore xattr use omap = true 2026-02-22T21:35:47.111 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:35:47.112 INFO:teuthology.orchestra.run.trial004.stdout: mon clock drift allowed = 1.000 2026-02-22T21:35:47.112 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:35:47.112 INFO:teuthology.orchestra.run.trial004.stdout: osd crush chooseleaf type = 0 2026-02-22T21:35:47.112 INFO:teuthology.orchestra.run.trial004.stdout: auth debug = true 2026-02-22T21:35:47.112 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:35:47.112 INFO:teuthology.orchestra.run.trial004.stdout: ms die on old message = true 2026-02-22T21:35:47.112 INFO:teuthology.orchestra.run.trial004.stdout: ms die on bug = true 2026-02-22T21:35:47.112 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:35:47.112 INFO:teuthology.orchestra.run.trial004.stdout: mon max pg per osd = 10000 # >= luminous 2026-02-22T21:35:47.112 INFO:teuthology.orchestra.run.trial004.stdout: mon pg warn max object skew = 0 2026-02-22T21:35:47.112 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:35:47.113 INFO:teuthology.orchestra.run.trial004.stdout: # disable pg_autoscaler by default for new pools 2026-02-22T21:35:47.113 INFO:teuthology.orchestra.run.trial004.stdout: osd_pool_default_pg_autoscale_mode = off 2026-02-22T21:35:47.113 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:35:47.113 INFO:teuthology.orchestra.run.trial004.stdout: osd pool default size = 2 2026-02-22T21:35:47.113 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:35:47.113 INFO:teuthology.orchestra.run.trial004.stdout: mon osd allow primary affinity = true 2026-02-22T21:35:47.113 INFO:teuthology.orchestra.run.trial004.stdout: mon osd allow pg remap = true 2026-02-22T21:35:47.113 INFO:teuthology.orchestra.run.trial004.stdout: mon warn on legacy crush tunables = false 2026-02-22T21:35:47.113 INFO:teuthology.orchestra.run.trial004.stdout: mon warn on crush straw calc version zero = false 2026-02-22T21:35:47.113 INFO:teuthology.orchestra.run.trial004.stdout: mon warn on no sortbitwise = false 2026-02-22T21:35:47.113 INFO:teuthology.orchestra.run.trial004.stdout: mon warn on osd down out interval zero = false 2026-02-22T21:35:47.113 INFO:teuthology.orchestra.run.trial004.stdout: mon warn on too few osds = false 2026-02-22T21:35:47.114 INFO:teuthology.orchestra.run.trial004.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2026-02-22T21:35:47.114 INFO:teuthology.orchestra.run.trial004.stdout: mon_warn_on_pool_no_redundancy = false 2026-02-22T21:35:47.114 INFO:teuthology.orchestra.run.trial004.stdout: mon_allow_pool_size_one = true 2026-02-22T21:35:47.114 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:35:47.114 INFO:teuthology.orchestra.run.trial004.stdout: osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2026-02-22T21:35:47.114 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:35:47.114 INFO:teuthology.orchestra.run.trial004.stdout: osd default data pool replay window = 5 2026-02-22T21:35:47.114 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:35:47.114 INFO:teuthology.orchestra.run.trial004.stdout: mon allow pool delete = true 2026-02-22T21:35:47.114 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:35:47.114 INFO:teuthology.orchestra.run.trial004.stdout: mon cluster log file level = debug 2026-02-22T21:35:47.114 INFO:teuthology.orchestra.run.trial004.stdout: debug asserts on shutdown = true 2026-02-22T21:35:47.115 INFO:teuthology.orchestra.run.trial004.stdout: mon health detail to clog = false 2026-02-22T21:35:47.115 INFO:teuthology.orchestra.run.trial004.stdout: mon host = 10.20.193.4 2026-02-22T21:35:47.115 INFO:teuthology.orchestra.run.trial004.stdout: mon client directed command retry = 5 2026-02-22T21:35:47.115 INFO:teuthology.orchestra.run.trial004.stdout: mon warn on pool no app = False 2026-02-22T21:35:47.115 INFO:teuthology.orchestra.run.trial004.stdout: ms inject socket failures = 5000 2026-02-22T21:35:47.115 INFO:teuthology.orchestra.run.trial004.stdout: fsid = 54a0d3ac-2ee3-469c-a716-4416b2f462d9 2026-02-22T21:35:47.115 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:35:47.115 INFO:teuthology.orchestra.run.trial004.stdout:[osd] 2026-02-22T21:35:47.115 INFO:teuthology.orchestra.run.trial004.stdout: osd journal size = 100 2026-02-22T21:35:47.115 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:35:47.115 INFO:teuthology.orchestra.run.trial004.stdout: osd scrub load threshold = 5.0 2026-02-22T21:35:47.115 INFO:teuthology.orchestra.run.trial004.stdout: osd scrub max interval = 600 2026-02-22T21:35:47.116 INFO:teuthology.orchestra.run.trial004.stdout: osd mclock profile = high_recovery_ops 2026-02-22T21:35:47.116 INFO:teuthology.orchestra.run.trial004.stdout: osd mclock skip benchmark = true 2026-02-22T21:35:47.116 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:35:47.116 INFO:teuthology.orchestra.run.trial004.stdout: osd recover clone overlap = true 2026-02-22T21:35:47.116 INFO:teuthology.orchestra.run.trial004.stdout: osd recovery max chunk = 1048576 2026-02-22T21:35:47.116 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:35:47.116 INFO:teuthology.orchestra.run.trial004.stdout: osd debug shutdown = true 2026-02-22T21:35:47.116 INFO:teuthology.orchestra.run.trial004.stdout: osd debug op order = true 2026-02-22T21:35:47.116 INFO:teuthology.orchestra.run.trial004.stdout: osd debug verify stray on activate = true 2026-02-22T21:35:47.116 INFO:teuthology.orchestra.run.trial004.stdout: osd debug trim objects = true 2026-02-22T21:35:47.116 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:35:47.116 INFO:teuthology.orchestra.run.trial004.stdout: osd open classes on start = true 2026-02-22T21:35:47.117 INFO:teuthology.orchestra.run.trial004.stdout: osd debug pg log writeout = true 2026-02-22T21:35:47.117 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:35:47.117 INFO:teuthology.orchestra.run.trial004.stdout: osd deep scrub update digest min age = 30 2026-02-22T21:35:47.117 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:35:47.117 INFO:teuthology.orchestra.run.trial004.stdout: osd map max advance = 10 2026-02-22T21:35:47.117 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:35:47.117 INFO:teuthology.orchestra.run.trial004.stdout: journal zero on create = true 2026-02-22T21:35:47.117 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:35:47.117 INFO:teuthology.orchestra.run.trial004.stdout: filestore ondisk finisher threads = 3 2026-02-22T21:35:47.117 INFO:teuthology.orchestra.run.trial004.stdout: filestore apply finisher threads = 3 2026-02-22T21:35:47.117 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:35:47.117 INFO:teuthology.orchestra.run.trial004.stdout: bdev debug aio = true 2026-02-22T21:35:47.117 INFO:teuthology.orchestra.run.trial004.stdout: osd debug misdirected ops = true 2026-02-22T21:35:47.118 INFO:teuthology.orchestra.run.trial004.stdout: bluestore block size = 96636764160 2026-02-22T21:35:47.118 INFO:teuthology.orchestra.run.trial004.stdout: bluestore compression algorithm = lz4 2026-02-22T21:35:47.118 INFO:teuthology.orchestra.run.trial004.stdout: bluestore compression mode = aggressive 2026-02-22T21:35:47.118 INFO:teuthology.orchestra.run.trial004.stdout: bluestore fsck on mount = True 2026-02-22T21:35:47.118 INFO:teuthology.orchestra.run.trial004.stdout: debug bluefs = 1/20 2026-02-22T21:35:47.118 INFO:teuthology.orchestra.run.trial004.stdout: debug bluestore = 1/20 2026-02-22T21:35:47.118 INFO:teuthology.orchestra.run.trial004.stdout: debug ms = 1 2026-02-22T21:35:47.118 INFO:teuthology.orchestra.run.trial004.stdout: debug osd = 20 2026-02-22T21:35:47.118 INFO:teuthology.orchestra.run.trial004.stdout: debug rocksdb = 4/10 2026-02-22T21:35:47.118 INFO:teuthology.orchestra.run.trial004.stdout: mon osd backfillfull_ratio = 0.85 2026-02-22T21:35:47.118 INFO:teuthology.orchestra.run.trial004.stdout: mon osd full ratio = 0.9 2026-02-22T21:35:47.118 INFO:teuthology.orchestra.run.trial004.stdout: mon osd nearfull ratio = 0.8 2026-02-22T21:35:47.118 INFO:teuthology.orchestra.run.trial004.stdout: osd failsafe full ratio = 0.95 2026-02-22T21:35:47.119 INFO:teuthology.orchestra.run.trial004.stdout: osd objectstore = bluestore 2026-02-22T21:35:47.119 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:35:47.119 INFO:teuthology.orchestra.run.trial004.stdout:[mgr] 2026-02-22T21:35:47.119 INFO:teuthology.orchestra.run.trial004.stdout: debug ms = 1 2026-02-22T21:35:47.119 INFO:teuthology.orchestra.run.trial004.stdout: debug mgr = 20 2026-02-22T21:35:47.119 INFO:teuthology.orchestra.run.trial004.stdout: debug mon = 20 2026-02-22T21:35:47.119 INFO:teuthology.orchestra.run.trial004.stdout: debug auth = 20 2026-02-22T21:35:47.119 INFO:teuthology.orchestra.run.trial004.stdout: mon reweight min pgs per osd = 4 2026-02-22T21:35:47.119 INFO:teuthology.orchestra.run.trial004.stdout: mon reweight min bytes per osd = 10 2026-02-22T21:35:47.119 INFO:teuthology.orchestra.run.trial004.stdout: mgr/telemetry/nag = false 2026-02-22T21:35:47.119 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:35:47.119 INFO:teuthology.orchestra.run.trial004.stdout:[mon] 2026-02-22T21:35:47.120 INFO:teuthology.orchestra.run.trial004.stdout: debug ms = 1 2026-02-22T21:35:47.120 INFO:teuthology.orchestra.run.trial004.stdout: debug mon = 20 2026-02-22T21:35:47.120 INFO:teuthology.orchestra.run.trial004.stdout: debug paxos = 20 2026-02-22T21:35:47.120 INFO:teuthology.orchestra.run.trial004.stdout: debug auth = 20 2026-02-22T21:35:47.120 INFO:teuthology.orchestra.run.trial004.stdout: mon data avail warn = 5 2026-02-22T21:35:47.120 INFO:teuthology.orchestra.run.trial004.stdout: mon mgr mkfs grace = 240 2026-02-22T21:35:47.120 INFO:teuthology.orchestra.run.trial004.stdout: mon reweight min pgs per osd = 4 2026-02-22T21:35:47.120 INFO:teuthology.orchestra.run.trial004.stdout: mon osd reporter subtree level = osd 2026-02-22T21:35:47.120 INFO:teuthology.orchestra.run.trial004.stdout: mon osd prime pg temp = true 2026-02-22T21:35:47.120 INFO:teuthology.orchestra.run.trial004.stdout: mon reweight min bytes per osd = 10 2026-02-22T21:35:47.120 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:35:47.120 INFO:teuthology.orchestra.run.trial004.stdout: # rotate auth tickets quickly to exercise renewal paths 2026-02-22T21:35:47.121 INFO:teuthology.orchestra.run.trial004.stdout: auth mon ticket ttl = 660 # 11m 2026-02-22T21:35:47.121 INFO:teuthology.orchestra.run.trial004.stdout: auth service ticket ttl = 240 # 4m 2026-02-22T21:35:47.121 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:35:47.121 INFO:teuthology.orchestra.run.trial004.stdout: # don't complain about insecure global_id in the test suite 2026-02-22T21:35:47.121 INFO:teuthology.orchestra.run.trial004.stdout: mon_warn_on_insecure_global_id_reclaim = false 2026-02-22T21:35:47.121 INFO:teuthology.orchestra.run.trial004.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2026-02-22T21:35:47.121 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:35:47.121 INFO:teuthology.orchestra.run.trial004.stdout: # 1m isn't quite enough 2026-02-22T21:35:47.121 INFO:teuthology.orchestra.run.trial004.stdout: mon_down_mkfs_grace = 2m 2026-02-22T21:35:47.121 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:35:47.121 INFO:teuthology.orchestra.run.trial004.stdout: mon_warn_on_filestore_osds = false 2026-02-22T21:35:47.121 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:35:47.122 INFO:teuthology.orchestra.run.trial004.stdout:[client] 2026-02-22T21:35:47.122 INFO:teuthology.orchestra.run.trial004.stdout: rgw cache enabled = true 2026-02-22T21:35:47.122 INFO:teuthology.orchestra.run.trial004.stdout: rgw enable ops log = true 2026-02-22T21:35:47.122 INFO:teuthology.orchestra.run.trial004.stdout: rgw enable usage log = true 2026-02-22T21:35:47.122 INFO:teuthology.orchestra.run.trial004.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2026-02-22T21:35:47.122 INFO:teuthology.orchestra.run.trial004.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2026-02-22T21:35:47.122 INFO:teuthology.orchestra.run.trial004.stdout: debug rbd = 20 2026-02-22T21:35:47.122 INFO:teuthology.orchestra.run.trial004.stdout: debug rbd_mirror = 15 2026-02-22T21:35:47.122 INFO:teuthology.orchestra.run.trial004.stdout: log to stderr = False 2026-02-22T21:35:47.122 INFO:teuthology.orchestra.run.trial004.stdout: rbd default features = 125 2026-02-22T21:35:47.122 INFO:teuthology.orchestra.run.trial004.stdout: rbd mirror image policy type = simple 2026-02-22T21:35:47.122 INFO:teuthology.orchestra.run.trial004.stdout:[mon.a] 2026-02-22T21:35:47.122 INFO:teuthology.orchestra.run.trial004.stdout:[client.mirror.0] 2026-02-22T21:35:47.123 INFO:teuthology.orchestra.run.trial004.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T21:35:47.123 INFO:teuthology.orchestra.run.trial004.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T21:35:47.123 INFO:teuthology.orchestra.run.trial004.stdout:[client.mirror.1] 2026-02-22T21:35:47.123 INFO:teuthology.orchestra.run.trial004.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T21:35:47.123 INFO:teuthology.orchestra.run.trial004.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T21:35:47.123 INFO:teuthology.orchestra.run.trial004.stdout:[client.mirror.2] 2026-02-22T21:35:47.123 INFO:teuthology.orchestra.run.trial004.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T21:35:47.123 INFO:teuthology.orchestra.run.trial004.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T21:35:47.123 INFO:teuthology.orchestra.run.trial004.stdout:[client.mirror.3] 2026-02-22T21:35:47.123 INFO:teuthology.orchestra.run.trial004.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T21:35:47.124 INFO:teuthology.orchestra.run.trial004.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T21:35:47.125 INFO:teuthology.orchestra.run.trial026.stdout:[global] 2026-02-22T21:35:47.126 INFO:teuthology.orchestra.run.trial026.stdout: chdir = "" 2026-02-22T21:35:47.126 INFO:teuthology.orchestra.run.trial026.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2026-02-22T21:35:47.126 INFO:teuthology.orchestra.run.trial026.stdout: auth supported = cephx 2026-02-22T21:35:47.126 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:35:47.126 INFO:teuthology.orchestra.run.trial026.stdout: filestore xattr use omap = true 2026-02-22T21:35:47.126 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:35:47.126 INFO:teuthology.orchestra.run.trial026.stdout: mon clock drift allowed = 1.000 2026-02-22T21:35:47.126 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:35:47.126 INFO:teuthology.orchestra.run.trial026.stdout: osd crush chooseleaf type = 0 2026-02-22T21:35:47.126 INFO:teuthology.orchestra.run.trial026.stdout: auth debug = true 2026-02-22T21:35:47.127 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:35:47.127 INFO:teuthology.orchestra.run.trial026.stdout: ms die on old message = true 2026-02-22T21:35:47.127 INFO:teuthology.orchestra.run.trial026.stdout: ms die on bug = true 2026-02-22T21:35:47.127 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:35:47.127 INFO:teuthology.orchestra.run.trial026.stdout: mon max pg per osd = 10000 # >= luminous 2026-02-22T21:35:47.127 INFO:teuthology.orchestra.run.trial026.stdout: mon pg warn max object skew = 0 2026-02-22T21:35:47.127 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:35:47.127 INFO:teuthology.orchestra.run.trial026.stdout: # disable pg_autoscaler by default for new pools 2026-02-22T21:35:47.127 INFO:teuthology.orchestra.run.trial026.stdout: osd_pool_default_pg_autoscale_mode = off 2026-02-22T21:35:47.127 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:35:47.127 INFO:teuthology.orchestra.run.trial026.stdout: osd pool default size = 2 2026-02-22T21:35:47.128 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:35:47.128 INFO:teuthology.orchestra.run.trial026.stdout: mon osd allow primary affinity = true 2026-02-22T21:35:47.128 INFO:teuthology.orchestra.run.trial026.stdout: mon osd allow pg remap = true 2026-02-22T21:35:47.128 INFO:teuthology.orchestra.run.trial026.stdout: mon warn on legacy crush tunables = false 2026-02-22T21:35:47.128 INFO:teuthology.orchestra.run.trial026.stdout: mon warn on crush straw calc version zero = false 2026-02-22T21:35:47.128 INFO:teuthology.orchestra.run.trial026.stdout: mon warn on no sortbitwise = false 2026-02-22T21:35:47.128 INFO:teuthology.orchestra.run.trial026.stdout: mon warn on osd down out interval zero = false 2026-02-22T21:35:47.128 INFO:teuthology.orchestra.run.trial026.stdout: mon warn on too few osds = false 2026-02-22T21:35:47.128 INFO:teuthology.orchestra.run.trial026.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2026-02-22T21:35:47.128 INFO:teuthology.orchestra.run.trial026.stdout: mon_warn_on_pool_no_redundancy = false 2026-02-22T21:35:47.128 INFO:teuthology.orchestra.run.trial026.stdout: mon_allow_pool_size_one = true 2026-02-22T21:35:47.128 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:35:47.129 INFO:teuthology.orchestra.run.trial026.stdout: osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2026-02-22T21:35:47.129 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:35:47.129 INFO:teuthology.orchestra.run.trial026.stdout: osd default data pool replay window = 5 2026-02-22T21:35:47.129 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:35:47.129 INFO:teuthology.orchestra.run.trial026.stdout: mon allow pool delete = true 2026-02-22T21:35:47.129 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:35:47.129 INFO:teuthology.orchestra.run.trial026.stdout: mon cluster log file level = debug 2026-02-22T21:35:47.129 INFO:teuthology.orchestra.run.trial026.stdout: debug asserts on shutdown = true 2026-02-22T21:35:47.129 INFO:teuthology.orchestra.run.trial026.stdout: mon health detail to clog = false 2026-02-22T21:35:47.129 INFO:teuthology.orchestra.run.trial026.stdout: mon host = 10.20.193.4 2026-02-22T21:35:47.129 INFO:teuthology.orchestra.run.trial026.stdout: mon client directed command retry = 5 2026-02-22T21:35:47.129 INFO:teuthology.orchestra.run.trial026.stdout: mon warn on pool no app = False 2026-02-22T21:35:47.129 INFO:teuthology.orchestra.run.trial026.stdout: ms inject socket failures = 5000 2026-02-22T21:35:47.130 INFO:teuthology.orchestra.run.trial026.stdout: fsid = 54a0d3ac-2ee3-469c-a716-4416b2f462d9 2026-02-22T21:35:47.130 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:35:47.130 INFO:teuthology.orchestra.run.trial026.stdout:[osd] 2026-02-22T21:35:47.130 INFO:teuthology.orchestra.run.trial026.stdout: osd journal size = 100 2026-02-22T21:35:47.130 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:35:47.130 INFO:teuthology.orchestra.run.trial026.stdout: osd scrub load threshold = 5.0 2026-02-22T21:35:47.130 INFO:teuthology.orchestra.run.trial026.stdout: osd scrub max interval = 600 2026-02-22T21:35:47.130 INFO:teuthology.orchestra.run.trial026.stdout: osd mclock profile = high_recovery_ops 2026-02-22T21:35:47.130 INFO:teuthology.orchestra.run.trial026.stdout: osd mclock skip benchmark = true 2026-02-22T21:35:47.130 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:35:47.130 INFO:teuthology.orchestra.run.trial026.stdout: osd recover clone overlap = true 2026-02-22T21:35:47.130 INFO:teuthology.orchestra.run.trial026.stdout: osd recovery max chunk = 1048576 2026-02-22T21:35:47.130 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:35:47.131 INFO:teuthology.orchestra.run.trial026.stdout: osd debug shutdown = true 2026-02-22T21:35:47.131 INFO:teuthology.orchestra.run.trial026.stdout: osd debug op order = true 2026-02-22T21:35:47.131 INFO:teuthology.orchestra.run.trial026.stdout: osd debug verify stray on activate = true 2026-02-22T21:35:47.131 INFO:teuthology.orchestra.run.trial026.stdout: osd debug trim objects = true 2026-02-22T21:35:47.131 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:35:47.131 INFO:teuthology.orchestra.run.trial026.stdout: osd open classes on start = true 2026-02-22T21:35:47.131 INFO:teuthology.orchestra.run.trial026.stdout: osd debug pg log writeout = true 2026-02-22T21:35:47.131 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:35:47.131 INFO:teuthology.orchestra.run.trial026.stdout: osd deep scrub update digest min age = 30 2026-02-22T21:35:47.131 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:35:47.131 INFO:teuthology.orchestra.run.trial026.stdout: osd map max advance = 10 2026-02-22T21:35:47.131 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:35:47.132 INFO:teuthology.orchestra.run.trial026.stdout: journal zero on create = true 2026-02-22T21:35:47.132 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:35:47.132 INFO:teuthology.orchestra.run.trial026.stdout: filestore ondisk finisher threads = 3 2026-02-22T21:35:47.132 INFO:teuthology.orchestra.run.trial026.stdout: filestore apply finisher threads = 3 2026-02-22T21:35:47.132 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:35:47.132 INFO:teuthology.orchestra.run.trial026.stdout: bdev debug aio = true 2026-02-22T21:35:47.132 INFO:teuthology.orchestra.run.trial026.stdout: osd debug misdirected ops = true 2026-02-22T21:35:47.132 INFO:teuthology.orchestra.run.trial026.stdout: bluestore block size = 96636764160 2026-02-22T21:35:47.132 INFO:teuthology.orchestra.run.trial026.stdout: bluestore compression algorithm = lz4 2026-02-22T21:35:47.132 INFO:teuthology.orchestra.run.trial026.stdout: bluestore compression mode = aggressive 2026-02-22T21:35:47.132 INFO:teuthology.orchestra.run.trial026.stdout: bluestore fsck on mount = True 2026-02-22T21:35:47.132 INFO:teuthology.orchestra.run.trial026.stdout: debug bluefs = 1/20 2026-02-22T21:35:47.132 INFO:teuthology.orchestra.run.trial026.stdout: debug bluestore = 1/20 2026-02-22T21:35:47.133 INFO:teuthology.orchestra.run.trial026.stdout: debug ms = 1 2026-02-22T21:35:47.133 INFO:teuthology.orchestra.run.trial026.stdout: debug osd = 20 2026-02-22T21:35:47.133 INFO:teuthology.orchestra.run.trial026.stdout: debug rocksdb = 4/10 2026-02-22T21:35:47.133 INFO:teuthology.orchestra.run.trial026.stdout: mon osd backfillfull_ratio = 0.85 2026-02-22T21:35:47.133 INFO:teuthology.orchestra.run.trial026.stdout: mon osd full ratio = 0.9 2026-02-22T21:35:47.133 INFO:teuthology.orchestra.run.trial026.stdout: mon osd nearfull ratio = 0.8 2026-02-22T21:35:47.133 INFO:teuthology.orchestra.run.trial026.stdout: osd failsafe full ratio = 0.95 2026-02-22T21:35:47.133 INFO:teuthology.orchestra.run.trial026.stdout: osd objectstore = bluestore 2026-02-22T21:35:47.133 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:35:47.133 INFO:teuthology.orchestra.run.trial026.stdout:[mgr] 2026-02-22T21:35:47.133 INFO:teuthology.orchestra.run.trial026.stdout: debug ms = 1 2026-02-22T21:35:47.133 INFO:teuthology.orchestra.run.trial026.stdout: debug mgr = 20 2026-02-22T21:35:47.133 INFO:teuthology.orchestra.run.trial026.stdout: debug mon = 20 2026-02-22T21:35:47.134 INFO:teuthology.orchestra.run.trial026.stdout: debug auth = 20 2026-02-22T21:35:47.134 INFO:teuthology.orchestra.run.trial026.stdout: mon reweight min pgs per osd = 4 2026-02-22T21:35:47.134 INFO:teuthology.orchestra.run.trial026.stdout: mon reweight min bytes per osd = 10 2026-02-22T21:35:47.134 INFO:teuthology.orchestra.run.trial026.stdout: mgr/telemetry/nag = false 2026-02-22T21:35:47.134 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:35:47.134 INFO:teuthology.orchestra.run.trial026.stdout:[mon] 2026-02-22T21:35:47.134 INFO:teuthology.orchestra.run.trial026.stdout: debug ms = 1 2026-02-22T21:35:47.134 INFO:teuthology.orchestra.run.trial026.stdout: debug mon = 20 2026-02-22T21:35:47.134 INFO:teuthology.orchestra.run.trial026.stdout: debug paxos = 20 2026-02-22T21:35:47.134 INFO:teuthology.orchestra.run.trial026.stdout: debug auth = 20 2026-02-22T21:35:47.134 INFO:teuthology.orchestra.run.trial026.stdout: mon data avail warn = 5 2026-02-22T21:35:47.134 INFO:teuthology.orchestra.run.trial026.stdout: mon mgr mkfs grace = 240 2026-02-22T21:35:47.134 INFO:teuthology.orchestra.run.trial026.stdout: mon reweight min pgs per osd = 4 2026-02-22T21:35:47.135 INFO:teuthology.orchestra.run.trial026.stdout: mon osd reporter subtree level = osd 2026-02-22T21:35:47.135 INFO:teuthology.orchestra.run.trial026.stdout: mon osd prime pg temp = true 2026-02-22T21:35:47.135 INFO:teuthology.orchestra.run.trial026.stdout: mon reweight min bytes per osd = 10 2026-02-22T21:35:47.135 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:35:47.135 INFO:teuthology.orchestra.run.trial026.stdout: # rotate auth tickets quickly to exercise renewal paths 2026-02-22T21:35:47.135 INFO:teuthology.orchestra.run.trial026.stdout: auth mon ticket ttl = 660 # 11m 2026-02-22T21:35:47.135 INFO:teuthology.orchestra.run.trial026.stdout: auth service ticket ttl = 240 # 4m 2026-02-22T21:35:47.135 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:35:47.135 INFO:teuthology.orchestra.run.trial026.stdout: # don't complain about insecure global_id in the test suite 2026-02-22T21:35:47.135 INFO:teuthology.orchestra.run.trial026.stdout: mon_warn_on_insecure_global_id_reclaim = false 2026-02-22T21:35:47.135 INFO:teuthology.orchestra.run.trial026.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2026-02-22T21:35:47.135 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:35:47.136 INFO:teuthology.orchestra.run.trial026.stdout: # 1m isn't quite enough 2026-02-22T21:35:47.136 INFO:teuthology.orchestra.run.trial026.stdout: mon_down_mkfs_grace = 2m 2026-02-22T21:35:47.136 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:35:47.136 INFO:teuthology.orchestra.run.trial026.stdout: mon_warn_on_filestore_osds = false 2026-02-22T21:35:47.136 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:35:47.136 INFO:teuthology.orchestra.run.trial026.stdout:[client] 2026-02-22T21:35:47.136 INFO:teuthology.orchestra.run.trial026.stdout: rgw cache enabled = true 2026-02-22T21:35:47.136 INFO:teuthology.orchestra.run.trial026.stdout: rgw enable ops log = true 2026-02-22T21:35:47.136 INFO:teuthology.orchestra.run.trial026.stdout: rgw enable usage log = true 2026-02-22T21:35:47.136 INFO:teuthology.orchestra.run.trial026.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2026-02-22T21:35:47.136 INFO:teuthology.orchestra.run.trial026.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2026-02-22T21:35:47.136 INFO:teuthology.orchestra.run.trial026.stdout: debug rbd = 20 2026-02-22T21:35:47.136 INFO:teuthology.orchestra.run.trial026.stdout: debug rbd_mirror = 15 2026-02-22T21:35:47.137 INFO:teuthology.orchestra.run.trial026.stdout: log to stderr = False 2026-02-22T21:35:47.137 INFO:teuthology.orchestra.run.trial026.stdout: rbd default features = 125 2026-02-22T21:35:47.137 INFO:teuthology.orchestra.run.trial026.stdout: rbd mirror image policy type = simple 2026-02-22T21:35:47.137 INFO:teuthology.orchestra.run.trial026.stdout:[mon.a] 2026-02-22T21:35:47.137 INFO:teuthology.orchestra.run.trial026.stdout:[client.mirror.0] 2026-02-22T21:35:47.137 INFO:teuthology.orchestra.run.trial026.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T21:35:47.137 INFO:teuthology.orchestra.run.trial026.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T21:35:47.137 INFO:teuthology.orchestra.run.trial026.stdout:[client.mirror.1] 2026-02-22T21:35:47.137 INFO:teuthology.orchestra.run.trial026.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T21:35:47.137 INFO:teuthology.orchestra.run.trial026.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T21:35:47.137 INFO:teuthology.orchestra.run.trial026.stdout:[client.mirror.2] 2026-02-22T21:35:47.137 INFO:teuthology.orchestra.run.trial026.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T21:35:47.137 INFO:teuthology.orchestra.run.trial026.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T21:35:47.138 INFO:teuthology.orchestra.run.trial026.stdout:[client.mirror.3] 2026-02-22T21:35:47.138 INFO:teuthology.orchestra.run.trial026.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T21:35:47.138 INFO:teuthology.orchestra.run.trial026.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T21:35:47.138 INFO:tasks.ceph:Creating admin key on cluster1.mon.a... 2026-02-22T21:35:47.138 DEBUG:teuthology.orchestra.run.trial004:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *' /etc/ceph/cluster1.keyring 2026-02-22T21:35:47.208 INFO:tasks.ceph:Copying monmap to all nodes... 2026-02-22T21:35:47.208 DEBUG:teuthology.orchestra.run.trial004:> set -ex 2026-02-22T21:35:47.209 DEBUG:teuthology.orchestra.run.trial004:> dd if=/etc/ceph/cluster1.keyring of=/dev/stdout 2026-02-22T21:35:47.267 DEBUG:teuthology.orchestra.run.trial004:> set -ex 2026-02-22T21:35:47.267 DEBUG:teuthology.orchestra.run.trial004:> dd if=/home/ubuntu/cephtest/cluster1.monmap of=/dev/stdout 2026-02-22T21:35:47.324 INFO:tasks.ceph:Sending monmap to node ubuntu@trial004.front.sepia.ceph.com 2026-02-22T21:35:47.324 DEBUG:teuthology.orchestra.run.trial004:> set -ex 2026-02-22T21:35:47.325 DEBUG:teuthology.orchestra.run.trial004:> sudo dd of=/etc/ceph/cluster1.keyring 2026-02-22T21:35:47.325 DEBUG:teuthology.orchestra.run.trial004:> sudo chmod 0644 /etc/ceph/cluster1.keyring 2026-02-22T21:35:47.399 DEBUG:teuthology.orchestra.run.trial004:> set -ex 2026-02-22T21:35:47.400 DEBUG:teuthology.orchestra.run.trial004:> dd of=/home/ubuntu/cephtest/cluster1.monmap 2026-02-22T21:35:47.456 INFO:tasks.ceph:Sending monmap to node ubuntu@trial026.front.sepia.ceph.com 2026-02-22T21:35:47.456 DEBUG:teuthology.orchestra.run.trial026:> set -ex 2026-02-22T21:35:47.456 DEBUG:teuthology.orchestra.run.trial026:> sudo dd of=/etc/ceph/cluster1.keyring 2026-02-22T21:35:47.456 DEBUG:teuthology.orchestra.run.trial026:> sudo chmod 0644 /etc/ceph/cluster1.keyring 2026-02-22T21:35:47.490 DEBUG:teuthology.orchestra.run.trial026:> set -ex 2026-02-22T21:35:47.490 DEBUG:teuthology.orchestra.run.trial026:> dd of=/home/ubuntu/cephtest/cluster1.monmap 2026-02-22T21:35:47.547 INFO:tasks.ceph:Setting up mon nodes... 2026-02-22T21:35:47.548 INFO:tasks.ceph:Setting up mgr nodes... 2026-02-22T21:35:47.548 DEBUG:teuthology.orchestra.run.trial004:> sudo mkdir -p /var/lib/ceph/mgr/cluster1-x && sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=mgr.x /var/lib/ceph/mgr/cluster1-x/keyring 2026-02-22T21:35:47.592 INFO:teuthology.orchestra.run.trial004.stdout:creating /var/lib/ceph/mgr/cluster1-x/keyring 2026-02-22T21:35:47.595 INFO:tasks.ceph:Setting up mds nodes... 2026-02-22T21:35:47.595 INFO:tasks.ceph_client:Setting up client nodes... 2026-02-22T21:35:47.596 DEBUG:teuthology.orchestra.run.trial004:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.0 /etc/ceph/cluster1.client.0.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.0.keyring 2026-02-22T21:35:47.668 INFO:teuthology.orchestra.run.trial004.stdout:creating /etc/ceph/cluster1.client.0.keyring 2026-02-22T21:35:47.680 DEBUG:teuthology.orchestra.run.trial026:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror /etc/ceph/cluster1.client.mirror.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.keyring 2026-02-22T21:35:47.717 INFO:teuthology.orchestra.run.trial026.stdout:creating /etc/ceph/cluster1.client.mirror.keyring 2026-02-22T21:35:47.729 DEBUG:teuthology.orchestra.run.trial026:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.0 /etc/ceph/cluster1.client.mirror.0.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.0.keyring 2026-02-22T21:35:47.804 INFO:teuthology.orchestra.run.trial026.stdout:creating /etc/ceph/cluster1.client.mirror.0.keyring 2026-02-22T21:35:47.815 DEBUG:teuthology.orchestra.run.trial026:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.1 /etc/ceph/cluster1.client.mirror.1.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.1.keyring 2026-02-22T21:35:47.892 INFO:teuthology.orchestra.run.trial026.stdout:creating /etc/ceph/cluster1.client.mirror.1.keyring 2026-02-22T21:35:47.903 DEBUG:teuthology.orchestra.run.trial026:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.2 /etc/ceph/cluster1.client.mirror.2.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.2.keyring 2026-02-22T21:35:47.976 INFO:teuthology.orchestra.run.trial026.stdout:creating /etc/ceph/cluster1.client.mirror.2.keyring 2026-02-22T21:35:47.987 DEBUG:teuthology.orchestra.run.trial026:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.3 /etc/ceph/cluster1.client.mirror.3.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.3.keyring 2026-02-22T21:35:48.063 INFO:teuthology.orchestra.run.trial026.stdout:creating /etc/ceph/cluster1.client.mirror.3.keyring 2026-02-22T21:35:48.074 DEBUG:teuthology.orchestra.run.trial026:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.4 /etc/ceph/cluster1.client.mirror.4.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.4.keyring 2026-02-22T21:35:48.148 INFO:teuthology.orchestra.run.trial026.stdout:creating /etc/ceph/cluster1.client.mirror.4.keyring 2026-02-22T21:35:48.159 DEBUG:teuthology.orchestra.run.trial026:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.5 /etc/ceph/cluster1.client.mirror.5.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.5.keyring 2026-02-22T21:35:48.235 INFO:teuthology.orchestra.run.trial026.stdout:creating /etc/ceph/cluster1.client.mirror.5.keyring 2026-02-22T21:35:48.246 DEBUG:teuthology.orchestra.run.trial026:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.6 /etc/ceph/cluster1.client.mirror.6.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.6.keyring 2026-02-22T21:35:48.319 INFO:teuthology.orchestra.run.trial026.stdout:creating /etc/ceph/cluster1.client.mirror.6.keyring 2026-02-22T21:35:48.330 INFO:tasks.ceph:Running mkfs on osd nodes... 2026-02-22T21:35:48.330 INFO:tasks.ceph:ctx.disk_config.remote_to_roles_to_dev: {Remote(name='ubuntu@trial004.front.sepia.ceph.com'): {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'}} 2026-02-22T21:35:48.331 DEBUG:teuthology.orchestra.run.trial004:> sudo mkdir -p /var/lib/ceph/osd/cluster1-0 2026-02-22T21:35:48.356 INFO:tasks.ceph:roles_to_devs: {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'} 2026-02-22T21:35:48.356 INFO:tasks.ceph:role: cluster1.osd.0 2026-02-22T21:35:48.356 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_1 on ubuntu@trial004.front.sepia.ceph.com 2026-02-22T21:35:48.356 DEBUG:teuthology.orchestra.run.trial004:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_1 2026-02-22T21:35:48.421 INFO:teuthology.orchestra.run.trial004.stdout:meta-data=/dev/vg_nvme/lv_1 isize=2048 agcount=4, agsize=45875200 blks 2026-02-22T21:35:48.421 INFO:teuthology.orchestra.run.trial004.stdout: = sectsz=4096 attr=2, projid32bit=1 2026-02-22T21:35:48.421 INFO:teuthology.orchestra.run.trial004.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2026-02-22T21:35:48.421 INFO:teuthology.orchestra.run.trial004.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2026-02-22T21:35:48.421 INFO:teuthology.orchestra.run.trial004.stdout:data = bsize=4096 blocks=183500800, imaxpct=25 2026-02-22T21:35:48.421 INFO:teuthology.orchestra.run.trial004.stdout: = sunit=0 swidth=0 blks 2026-02-22T21:35:48.421 INFO:teuthology.orchestra.run.trial004.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2026-02-22T21:35:48.421 INFO:teuthology.orchestra.run.trial004.stdout:log =internal log bsize=4096 blocks=89600, version=2 2026-02-22T21:35:48.421 INFO:teuthology.orchestra.run.trial004.stdout: = sectsz=4096 sunit=1 blks, lazy-count=1 2026-02-22T21:35:48.422 INFO:teuthology.orchestra.run.trial004.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2026-02-22T21:35:48.991 INFO:teuthology.orchestra.run.trial004.stdout:Discarding blocks...Done. 2026-02-22T21:35:48.993 INFO:tasks.ceph:mount /dev/vg_nvme/lv_1 on ubuntu@trial004.front.sepia.ceph.com -o noatime 2026-02-22T21:35:48.994 DEBUG:teuthology.orchestra.run.trial004:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_1 /var/lib/ceph/osd/cluster1-0 2026-02-22T21:35:49.061 DEBUG:teuthology.orchestra.run.trial004:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster1-0 2026-02-22T21:35:49.129 DEBUG:teuthology.orchestra.run.trial004:> sudo mkdir -p /var/lib/ceph/osd/cluster1-1 2026-02-22T21:35:49.195 INFO:tasks.ceph:roles_to_devs: {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'} 2026-02-22T21:35:49.195 INFO:tasks.ceph:role: cluster1.osd.1 2026-02-22T21:35:49.195 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_2 on ubuntu@trial004.front.sepia.ceph.com 2026-02-22T21:35:49.195 DEBUG:teuthology.orchestra.run.trial004:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_2 2026-02-22T21:35:49.262 INFO:teuthology.orchestra.run.trial004.stdout:meta-data=/dev/vg_nvme/lv_2 isize=2048 agcount=4, agsize=45875200 blks 2026-02-22T21:35:49.262 INFO:teuthology.orchestra.run.trial004.stdout: = sectsz=4096 attr=2, projid32bit=1 2026-02-22T21:35:49.262 INFO:teuthology.orchestra.run.trial004.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2026-02-22T21:35:49.262 INFO:teuthology.orchestra.run.trial004.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2026-02-22T21:35:49.263 INFO:teuthology.orchestra.run.trial004.stdout:data = bsize=4096 blocks=183500800, imaxpct=25 2026-02-22T21:35:49.263 INFO:teuthology.orchestra.run.trial004.stdout: = sunit=0 swidth=0 blks 2026-02-22T21:35:49.263 INFO:teuthology.orchestra.run.trial004.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2026-02-22T21:35:49.263 INFO:teuthology.orchestra.run.trial004.stdout:log =internal log bsize=4096 blocks=89600, version=2 2026-02-22T21:35:49.263 INFO:teuthology.orchestra.run.trial004.stdout: = sectsz=4096 sunit=1 blks, lazy-count=1 2026-02-22T21:35:49.263 INFO:teuthology.orchestra.run.trial004.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2026-02-22T21:35:49.820 INFO:teuthology.orchestra.run.trial004.stdout:Discarding blocks...Done. 2026-02-22T21:35:49.822 INFO:tasks.ceph:mount /dev/vg_nvme/lv_2 on ubuntu@trial004.front.sepia.ceph.com -o noatime 2026-02-22T21:35:49.822 DEBUG:teuthology.orchestra.run.trial004:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_2 /var/lib/ceph/osd/cluster1-1 2026-02-22T21:35:49.890 DEBUG:teuthology.orchestra.run.trial004:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster1-1 2026-02-22T21:35:49.956 DEBUG:teuthology.orchestra.run.trial004:> sudo mkdir -p /var/lib/ceph/osd/cluster1-2 2026-02-22T21:35:50.019 INFO:tasks.ceph:roles_to_devs: {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'} 2026-02-22T21:35:50.019 INFO:tasks.ceph:role: cluster1.osd.2 2026-02-22T21:35:50.019 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_3 on ubuntu@trial004.front.sepia.ceph.com 2026-02-22T21:35:50.020 DEBUG:teuthology.orchestra.run.trial004:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_3 2026-02-22T21:35:50.083 INFO:teuthology.orchestra.run.trial004.stdout:meta-data=/dev/vg_nvme/lv_3 isize=2048 agcount=4, agsize=45875200 blks 2026-02-22T21:35:50.084 INFO:teuthology.orchestra.run.trial004.stdout: = sectsz=4096 attr=2, projid32bit=1 2026-02-22T21:35:50.084 INFO:teuthology.orchestra.run.trial004.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2026-02-22T21:35:50.084 INFO:teuthology.orchestra.run.trial004.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2026-02-22T21:35:50.084 INFO:teuthology.orchestra.run.trial004.stdout:data = bsize=4096 blocks=183500800, imaxpct=25 2026-02-22T21:35:50.084 INFO:teuthology.orchestra.run.trial004.stdout: = sunit=0 swidth=0 blks 2026-02-22T21:35:50.085 INFO:teuthology.orchestra.run.trial004.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2026-02-22T21:35:50.085 INFO:teuthology.orchestra.run.trial004.stdout:log =internal log bsize=4096 blocks=89600, version=2 2026-02-22T21:35:50.085 INFO:teuthology.orchestra.run.trial004.stdout: = sectsz=4096 sunit=1 blks, lazy-count=1 2026-02-22T21:35:50.085 INFO:teuthology.orchestra.run.trial004.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2026-02-22T21:35:50.640 INFO:teuthology.orchestra.run.trial004.stdout:Discarding blocks...Done. 2026-02-22T21:35:50.642 INFO:tasks.ceph:mount /dev/vg_nvme/lv_3 on ubuntu@trial004.front.sepia.ceph.com -o noatime 2026-02-22T21:35:50.642 DEBUG:teuthology.orchestra.run.trial004:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_3 /var/lib/ceph/osd/cluster1-2 2026-02-22T21:35:50.710 DEBUG:teuthology.orchestra.run.trial004:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster1-2 2026-02-22T21:35:50.777 DEBUG:teuthology.orchestra.run.trial004:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster1 --mkfs --mkkey -i 0 --monmap /home/ubuntu/cephtest/cluster1.monmap 2026-02-22T21:35:50.856 INFO:teuthology.orchestra.run.trial004.stderr:2026-02-22T21:35:50.854+0000 7fba8da0c780 -1 auth: error reading file: /var/lib/ceph/osd/cluster1-0/keyring: can't open /var/lib/ceph/osd/cluster1-0/keyring: (2) No such file or directory 2026-02-22T21:35:50.856 INFO:teuthology.orchestra.run.trial004.stderr:2026-02-22T21:35:50.855+0000 7fba8da0c780 -1 created new key in keyring /var/lib/ceph/osd/cluster1-0/keyring 2026-02-22T21:35:50.857 INFO:teuthology.orchestra.run.trial004.stderr:2026-02-22T21:35:50.855+0000 7fba8da0c780 -1 bdev(0x55adb9307800 /var/lib/ceph/osd/cluster1-0/block) open stat got: (1) Operation not permitted 2026-02-22T21:35:50.857 INFO:teuthology.orchestra.run.trial004.stderr:2026-02-22T21:35:50.855+0000 7fba8da0c780 -1 bluestore(/var/lib/ceph/osd/cluster1-0) _read_fsid unparsable uuid 2026-02-22T21:35:53.765 DEBUG:teuthology.orchestra.run.trial004:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster1-0 2026-02-22T21:35:53.830 DEBUG:teuthology.orchestra.run.trial004:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster1 --mkfs --mkkey -i 1 --monmap /home/ubuntu/cephtest/cluster1.monmap 2026-02-22T21:35:53.909 INFO:teuthology.orchestra.run.trial004.stderr:2026-02-22T21:35:53.907+0000 7fbb2ffb4780 -1 auth: error reading file: /var/lib/ceph/osd/cluster1-1/keyring: can't open /var/lib/ceph/osd/cluster1-1/keyring: (2) No such file or directory 2026-02-22T21:35:53.909 INFO:teuthology.orchestra.run.trial004.stderr:2026-02-22T21:35:53.907+0000 7fbb2ffb4780 -1 created new key in keyring /var/lib/ceph/osd/cluster1-1/keyring 2026-02-22T21:35:53.909 INFO:teuthology.orchestra.run.trial004.stderr:2026-02-22T21:35:53.907+0000 7fbb2ffb4780 -1 bdev(0x55f980a31800 /var/lib/ceph/osd/cluster1-1/block) open stat got: (1) Operation not permitted 2026-02-22T21:35:53.909 INFO:teuthology.orchestra.run.trial004.stderr:2026-02-22T21:35:53.907+0000 7fbb2ffb4780 -1 bluestore(/var/lib/ceph/osd/cluster1-1) _read_fsid unparsable uuid 2026-02-22T21:35:56.814 DEBUG:teuthology.orchestra.run.trial004:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster1-1 2026-02-22T21:35:56.879 DEBUG:teuthology.orchestra.run.trial004:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster1 --mkfs --mkkey -i 2 --monmap /home/ubuntu/cephtest/cluster1.monmap 2026-02-22T21:35:56.958 INFO:teuthology.orchestra.run.trial004.stderr:2026-02-22T21:35:56.957+0000 7f8f30ad9780 -1 auth: error reading file: /var/lib/ceph/osd/cluster1-2/keyring: can't open /var/lib/ceph/osd/cluster1-2/keyring: (2) No such file or directory 2026-02-22T21:35:56.958 INFO:teuthology.orchestra.run.trial004.stderr:2026-02-22T21:35:56.957+0000 7f8f30ad9780 -1 created new key in keyring /var/lib/ceph/osd/cluster1-2/keyring 2026-02-22T21:35:56.958 INFO:teuthology.orchestra.run.trial004.stderr:2026-02-22T21:35:56.957+0000 7f8f30ad9780 -1 bdev(0x5635063ff800 /var/lib/ceph/osd/cluster1-2/block) open stat got: (1) Operation not permitted 2026-02-22T21:35:56.958 INFO:teuthology.orchestra.run.trial004.stderr:2026-02-22T21:35:56.957+0000 7f8f30ad9780 -1 bluestore(/var/lib/ceph/osd/cluster1-2) _read_fsid unparsable uuid 2026-02-22T21:36:00.122 DEBUG:teuthology.orchestra.run.trial004:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster1-2 2026-02-22T21:36:00.187 INFO:tasks.ceph:Reading keys from all nodes... 2026-02-22T21:36:00.187 DEBUG:teuthology.orchestra.run.trial004:> set -ex 2026-02-22T21:36:00.188 DEBUG:teuthology.orchestra.run.trial004:> sudo dd if=/var/lib/ceph/mgr/cluster1-x/keyring of=/dev/stdout 2026-02-22T21:36:00.251 DEBUG:teuthology.orchestra.run.trial004:> set -ex 2026-02-22T21:36:00.251 DEBUG:teuthology.orchestra.run.trial004:> sudo dd if=/var/lib/ceph/osd/cluster1-0/keyring of=/dev/stdout 2026-02-22T21:36:00.314 DEBUG:teuthology.orchestra.run.trial004:> set -ex 2026-02-22T21:36:00.314 DEBUG:teuthology.orchestra.run.trial004:> sudo dd if=/var/lib/ceph/osd/cluster1-1/keyring of=/dev/stdout 2026-02-22T21:36:00.377 DEBUG:teuthology.orchestra.run.trial004:> set -ex 2026-02-22T21:36:00.377 DEBUG:teuthology.orchestra.run.trial004:> sudo dd if=/var/lib/ceph/osd/cluster1-2/keyring of=/dev/stdout 2026-02-22T21:36:00.440 DEBUG:teuthology.orchestra.run.trial004:> set -ex 2026-02-22T21:36:00.440 DEBUG:teuthology.orchestra.run.trial004:> dd if=/etc/ceph/cluster1.client.0.keyring of=/dev/stdout 2026-02-22T21:36:00.497 DEBUG:teuthology.orchestra.run.trial026:> set -ex 2026-02-22T21:36:00.497 DEBUG:teuthology.orchestra.run.trial026:> dd if=/etc/ceph/cluster1.client.mirror.keyring of=/dev/stdout 2026-02-22T21:36:00.517 DEBUG:teuthology.orchestra.run.trial026:> set -ex 2026-02-22T21:36:00.517 DEBUG:teuthology.orchestra.run.trial026:> dd if=/etc/ceph/cluster1.client.mirror.0.keyring of=/dev/stdout 2026-02-22T21:36:00.575 DEBUG:teuthology.orchestra.run.trial026:> set -ex 2026-02-22T21:36:00.575 DEBUG:teuthology.orchestra.run.trial026:> dd if=/etc/ceph/cluster1.client.mirror.1.keyring of=/dev/stdout 2026-02-22T21:36:00.633 DEBUG:teuthology.orchestra.run.trial026:> set -ex 2026-02-22T21:36:00.633 DEBUG:teuthology.orchestra.run.trial026:> dd if=/etc/ceph/cluster1.client.mirror.2.keyring of=/dev/stdout 2026-02-22T21:36:00.690 DEBUG:teuthology.orchestra.run.trial026:> set -ex 2026-02-22T21:36:00.690 DEBUG:teuthology.orchestra.run.trial026:> dd if=/etc/ceph/cluster1.client.mirror.3.keyring of=/dev/stdout 2026-02-22T21:36:00.746 DEBUG:teuthology.orchestra.run.trial026:> set -ex 2026-02-22T21:36:00.746 DEBUG:teuthology.orchestra.run.trial026:> dd if=/etc/ceph/cluster1.client.mirror.4.keyring of=/dev/stdout 2026-02-22T21:36:00.803 DEBUG:teuthology.orchestra.run.trial026:> set -ex 2026-02-22T21:36:00.803 DEBUG:teuthology.orchestra.run.trial026:> dd if=/etc/ceph/cluster1.client.mirror.5.keyring of=/dev/stdout 2026-02-22T21:36:00.860 DEBUG:teuthology.orchestra.run.trial026:> set -ex 2026-02-22T21:36:00.860 DEBUG:teuthology.orchestra.run.trial026:> dd if=/etc/ceph/cluster1.client.mirror.6.keyring of=/dev/stdout 2026-02-22T21:36:00.917 INFO:tasks.ceph:Adding keys to all mons... 2026-02-22T21:36:00.918 DEBUG:teuthology.orchestra.run.trial004:> sudo tee -a /etc/ceph/cluster1.keyring 2026-02-22T21:36:00.941 INFO:teuthology.orchestra.run.trial004.stdout:[mgr.x] 2026-02-22T21:36:00.941 INFO:teuthology.orchestra.run.trial004.stdout: key = AQCzdptpVoVRIxAAxVY9t4Vm1P0qV5QPdQF+uQ== 2026-02-22T21:36:00.942 INFO:teuthology.orchestra.run.trial004.stdout:[osd.0] 2026-02-22T21:36:00.942 INFO:teuthology.orchestra.run.trial004.stdout: key = AQC2dptpl/cDMxAARjX6FbuR+hhQ51Ixp7zw7A== 2026-02-22T21:36:00.942 INFO:teuthology.orchestra.run.trial004.stdout:[osd.1] 2026-02-22T21:36:00.942 INFO:teuthology.orchestra.run.trial004.stdout: key = AQC5dptp4M4iNhAAyw939axcc7MNiAw0PEoXbw== 2026-02-22T21:36:00.942 INFO:teuthology.orchestra.run.trial004.stdout:[osd.2] 2026-02-22T21:36:00.942 INFO:teuthology.orchestra.run.trial004.stdout: key = AQC8dptp6lsdORAAILlbkeIdldSiz2Qet6znvw== 2026-02-22T21:36:00.943 INFO:teuthology.orchestra.run.trial004.stdout:[client.0] 2026-02-22T21:36:00.943 INFO:teuthology.orchestra.run.trial004.stdout: key = AQCzdptprv/YJxAAEgNSzqmsxVDWPUchYcL0Og== 2026-02-22T21:36:00.943 INFO:teuthology.orchestra.run.trial004.stdout:[client.mirror] 2026-02-22T21:36:00.943 INFO:teuthology.orchestra.run.trial004.stdout: key = AQCzdptpKJe9KhAApC05qr2GvyvojF3F7P0D2Q== 2026-02-22T21:36:00.943 INFO:teuthology.orchestra.run.trial004.stdout:[client.mirror.0] 2026-02-22T21:36:00.943 INFO:teuthology.orchestra.run.trial004.stdout: key = AQCzdptpipfiLxAAjzB/u0QwZJmrhV96Y++AEg== 2026-02-22T21:36:00.944 INFO:teuthology.orchestra.run.trial004.stdout:[client.mirror.1] 2026-02-22T21:36:00.944 INFO:teuthology.orchestra.run.trial004.stdout: key = AQCzdptpflQgNRAAtzGrTX/PFYfDsxh6+zboAw== 2026-02-22T21:36:00.944 INFO:teuthology.orchestra.run.trial004.stdout:[client.mirror.2] 2026-02-22T21:36:00.944 INFO:teuthology.orchestra.run.trial004.stdout: key = AQCzdptpibgiOhAABk6NsI44ocuIgNnj39mS+Q== 2026-02-22T21:36:00.944 INFO:teuthology.orchestra.run.trial004.stdout:[client.mirror.3] 2026-02-22T21:36:00.944 INFO:teuthology.orchestra.run.trial004.stdout: key = AQC0dptpPNy4AxAATEmDA2RB4Tnwt9QgC5zUUA== 2026-02-22T21:36:00.945 INFO:teuthology.orchestra.run.trial004.stdout:[client.mirror.4] 2026-02-22T21:36:00.945 INFO:teuthology.orchestra.run.trial004.stdout: key = AQC0dptp2KTTCBAAPXuOBoqcPz/sUpDg+OFCOw== 2026-02-22T21:36:00.945 INFO:teuthology.orchestra.run.trial004.stdout:[client.mirror.5] 2026-02-22T21:36:00.945 INFO:teuthology.orchestra.run.trial004.stdout: key = AQC0dptpcB74DRAAgLXOJXCXSWKqEk6ZzE02hg== 2026-02-22T21:36:00.945 INFO:teuthology.orchestra.run.trial004.stdout:[client.mirror.6] 2026-02-22T21:36:00.945 INFO:teuthology.orchestra.run.trial004.stdout: key = AQC0dptpN7QIExAA6QJeuxwm9viXT0wiYXu7rQ== 2026-02-22T21:36:00.947 DEBUG:teuthology.orchestra.run.trial004:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=mgr.x --cap mon 'allow profile mgr' --cap osd 'allow *' --cap mds 'allow *' 2026-02-22T21:36:01.027 DEBUG:teuthology.orchestra.run.trial004:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=osd.0 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2026-02-22T21:36:01.106 DEBUG:teuthology.orchestra.run.trial004:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=osd.1 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2026-02-22T21:36:01.185 DEBUG:teuthology.orchestra.run.trial004:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=osd.2 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2026-02-22T21:36:01.264 DEBUG:teuthology.orchestra.run.trial004:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.0 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T21:36:01.344 DEBUG:teuthology.orchestra.run.trial004:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T21:36:01.427 DEBUG:teuthology.orchestra.run.trial004:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.0 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T21:36:01.507 DEBUG:teuthology.orchestra.run.trial004:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.1 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T21:36:01.587 DEBUG:teuthology.orchestra.run.trial004:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.2 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T21:36:01.668 DEBUG:teuthology.orchestra.run.trial004:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.3 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T21:36:01.747 DEBUG:teuthology.orchestra.run.trial004:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.4 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T21:36:01.828 DEBUG:teuthology.orchestra.run.trial004:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.5 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T21:36:01.909 DEBUG:teuthology.orchestra.run.trial004:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.6 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T21:36:01.991 INFO:tasks.ceph:Running mkfs on mon nodes... 2026-02-22T21:36:01.991 DEBUG:teuthology.orchestra.run.trial004:> sudo mkdir -p /var/lib/ceph/mon/cluster1-a 2026-02-22T21:36:02.055 DEBUG:teuthology.orchestra.run.trial004:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-mon --cluster cluster1 --mkfs -i a --monmap /home/ubuntu/cephtest/cluster1.monmap --keyring /etc/ceph/cluster1.keyring 2026-02-22T21:36:02.141 DEBUG:teuthology.orchestra.run.trial004:> sudo chown -R ceph:ceph /var/lib/ceph/mon/cluster1-a 2026-02-22T21:36:02.166 DEBUG:teuthology.orchestra.run.trial004:> rm -- /home/ubuntu/cephtest/cluster1.monmap 2026-02-22T21:36:02.221 INFO:tasks.ceph:Starting mon daemons in cluster cluster1... 2026-02-22T21:36:02.222 INFO:tasks.ceph.cluster1.mon.a:Restarting daemon 2026-02-22T21:36:02.222 DEBUG:teuthology.orchestra.run.trial004:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mon -f --cluster cluster1 -i a 2026-02-22T21:36:02.265 INFO:tasks.ceph.cluster1.mon.a:Started 2026-02-22T21:36:02.265 INFO:tasks.ceph:Starting mgr daemons in cluster cluster1... 2026-02-22T21:36:02.266 INFO:tasks.ceph.cluster1.mgr.x:Restarting daemon 2026-02-22T21:36:02.266 DEBUG:teuthology.orchestra.run.trial004:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mgr -f --cluster cluster1 -i x 2026-02-22T21:36:02.269 INFO:tasks.ceph.cluster1.mgr.x:Started 2026-02-22T21:36:02.270 INFO:tasks.ceph:Setting crush tunables to default 2026-02-22T21:36:02.270 DEBUG:teuthology.orchestra.run.trial004:> sudo ceph --cluster cluster1 osd crush tunables default 2026-02-22T21:36:02.372 INFO:teuthology.orchestra.run.trial004.stderr:adjusted tunables profile to default 2026-02-22T21:36:02.380 INFO:tasks.ceph:check_enable_crimson: False 2026-02-22T21:36:02.380 INFO:tasks.ceph:Starting osd daemons in cluster cluster1... 2026-02-22T21:36:02.380 DEBUG:teuthology.orchestra.run.trial004:> set -ex 2026-02-22T21:36:02.380 DEBUG:teuthology.orchestra.run.trial004:> sudo dd if=/var/lib/ceph/osd/cluster1-0/fsid of=/dev/stdout 2026-02-22T21:36:02.405 DEBUG:teuthology.orchestra.run.trial004:> set -ex 2026-02-22T21:36:02.405 DEBUG:teuthology.orchestra.run.trial004:> sudo dd if=/var/lib/ceph/osd/cluster1-1/fsid of=/dev/stdout 2026-02-22T21:36:02.469 DEBUG:teuthology.orchestra.run.trial004:> set -ex 2026-02-22T21:36:02.469 DEBUG:teuthology.orchestra.run.trial004:> sudo dd if=/var/lib/ceph/osd/cluster1-2/fsid of=/dev/stdout 2026-02-22T21:36:02.533 DEBUG:teuthology.orchestra.run.trial004:> sudo ceph --cluster cluster1 osd new ad69c917-7b9d-4a6d-bd05-3187fef2ebef 0 2026-02-22T21:36:02.675 INFO:teuthology.orchestra.run.trial004.stdout:0 2026-02-22T21:36:02.682 DEBUG:teuthology.orchestra.run.trial004:> sudo ceph --cluster cluster1 osd new 8e27aac6-4677-41a2-8575-2b92ead46c7e 1 2026-02-22T21:36:02.785 INFO:teuthology.orchestra.run.trial004.stdout:1 2026-02-22T21:36:02.794 DEBUG:teuthology.orchestra.run.trial004:> sudo ceph --cluster cluster1 osd new 28f6fce7-f1df-448c-91e9-b0eb54f0ff5f 2 2026-02-22T21:36:02.875 INFO:tasks.ceph.cluster1.mgr.x.trial004.stderr:2026-02-22T21:36:02.874+0000 7f71ce7d8140 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2026-02-22T21:36:02.895 INFO:teuthology.orchestra.run.trial004.stdout:2 2026-02-22T21:36:02.903 INFO:tasks.ceph.cluster1.osd.0:Restarting daemon 2026-02-22T21:36:02.904 DEBUG:teuthology.orchestra.run.trial004:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster1 -i 0 2026-02-22T21:36:02.906 INFO:tasks.ceph.cluster1.osd.0:Started 2026-02-22T21:36:02.906 INFO:tasks.ceph.cluster1.osd.1:Restarting daemon 2026-02-22T21:36:02.906 DEBUG:teuthology.orchestra.run.trial004:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster1 -i 1 2026-02-22T21:36:02.908 INFO:tasks.ceph.cluster1.osd.1:Started 2026-02-22T21:36:02.908 INFO:tasks.ceph.cluster1.osd.2:Restarting daemon 2026-02-22T21:36:02.908 DEBUG:teuthology.orchestra.run.trial004:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster1 -i 2 2026-02-22T21:36:02.910 INFO:tasks.ceph.cluster1.osd.2:Started 2026-02-22T21:36:02.911 DEBUG:teuthology.orchestra.run.trial004:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd dump --format=json 2026-02-22T21:36:03.001 INFO:tasks.ceph.cluster1.mgr.x.trial004.stderr:2026-02-22T21:36:02.999+0000 7f71ce7d8140 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2026-02-22T21:36:03.010 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:36:03.011 INFO:teuthology.orchestra.run.trial004.stdout:{"epoch":5,"fsid":"54a0d3ac-2ee3-469c-a716-4416b2f462d9","created":"2026-02-22T21:36:02.327898+0000","modified":"2026-02-22T21:36:02.895351+0000","last_up_change":"0.000000","last_in_change":"2026-02-22T21:36:02.895351+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"ad69c917-7b9d-4a6d-bd05-3187fef2ebef","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"8e27aac6-4677-41a2-8575-2b92ead46c7e","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"28f6fce7-f1df-448c-91e9-b0eb54f0ff5f","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-22T21:36:03.017 INFO:tasks.ceph.ceph_manager.cluster1:[] 2026-02-22T21:36:03.017 INFO:tasks.ceph:Waiting for OSDs to come up 2026-02-22T21:36:03.028 INFO:tasks.ceph.cluster1.mgr.x.trial004.stderr:2026-02-22T21:36:03.027+0000 7f71ce7d8140 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2026-02-22T21:36:03.061 INFO:tasks.ceph.cluster1.mgr.x.trial004.stderr:2026-02-22T21:36:03.060+0000 7f71ce7d8140 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2026-02-22T21:36:03.084 INFO:tasks.ceph.cluster1.mgr.x.trial004.stderr:2026-02-22T21:36:03.083+0000 7f71ce7d8140 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2026-02-22T21:36:03.110 INFO:tasks.ceph.cluster1.mgr.x.trial004.stderr:2026-02-22T21:36:03.109+0000 7f71ce7d8140 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2026-02-22T21:36:03.166 INFO:tasks.ceph.cluster1.mgr.x.trial004.stderr:2026-02-22T21:36:03.164+0000 7f71ce7d8140 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2026-02-22T21:36:03.226 INFO:tasks.ceph.cluster1.mgr.x.trial004.stderr:/usr/lib64/python3.9/site-packages/scipy/__init__.py:73: UserWarning: NumPy was imported from a Python sub-interpreter but NumPy does not properly support sub-interpreters. This will likely work for most users but might cause hard to track down issues or subtle bugs. A common user of the rare sub-interpreter feature is wsgi which also allows single-interpreter mode. 2026-02-22T21:36:03.226 INFO:tasks.ceph.cluster1.mgr.x.trial004.stderr:Improvements in the case of bugs are welcome, but is not on the NumPy roadmap, and full support may require significant effort to achieve. 2026-02-22T21:36:03.226 INFO:tasks.ceph.cluster1.mgr.x.trial004.stderr: from numpy import show_config as show_numpy_config 2026-02-22T21:36:03.230 INFO:tasks.ceph.cluster1.mgr.x.trial004.stderr:2026-02-22T21:36:03.228+0000 7f71ce7d8140 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2026-02-22T21:36:03.255 INFO:tasks.ceph.cluster1.mgr.x.trial004.stderr:2026-02-22T21:36:03.254+0000 7f71ce7d8140 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2026-02-22T21:36:03.310 INFO:tasks.ceph.cluster1.mgr.x.trial004.stderr:2026-02-22T21:36:03.309+0000 7f71ce7d8140 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2026-02-22T21:36:03.320 DEBUG:teuthology.orchestra.run.trial004:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster1 osd dump --format=json 2026-02-22T21:36:03.352 INFO:tasks.ceph.cluster1.mgr.x.trial004.stderr:2026-02-22T21:36:03.351+0000 7f71ce7d8140 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2026-02-22T21:36:03.378 INFO:tasks.ceph.cluster1.mgr.x.trial004.stderr:2026-02-22T21:36:03.377+0000 7f71ce7d8140 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2026-02-22T21:36:03.403 INFO:tasks.ceph.cluster1.mgr.x.trial004.stderr:2026-02-22T21:36:03.402+0000 7f71ce7d8140 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2026-02-22T21:36:03.413 INFO:teuthology.misc.health.trial004.stdout: 2026-02-22T21:36:03.413 INFO:teuthology.misc.health.trial004.stdout:{"epoch":5,"fsid":"54a0d3ac-2ee3-469c-a716-4416b2f462d9","created":"2026-02-22T21:36:02.327898+0000","modified":"2026-02-22T21:36:02.895351+0000","last_up_change":"0.000000","last_in_change":"2026-02-22T21:36:02.895351+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"ad69c917-7b9d-4a6d-bd05-3187fef2ebef","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"8e27aac6-4677-41a2-8575-2b92ead46c7e","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"28f6fce7-f1df-448c-91e9-b0eb54f0ff5f","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-22T21:36:03.419 DEBUG:teuthology.misc:0 of 3 OSDs are up 2026-02-22T21:36:03.428 INFO:tasks.ceph.cluster1.mgr.x.trial004.stderr:2026-02-22T21:36:03.427+0000 7f71ce7d8140 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2026-02-22T21:36:03.451 INFO:tasks.ceph.cluster1.mgr.x.trial004.stderr:2026-02-22T21:36:03.450+0000 7f71ce7d8140 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2026-02-22T21:36:03.643 INFO:tasks.ceph.cluster1.mgr.x.trial004.stderr:2026-02-22T21:36:03.642+0000 7f71ce7d8140 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2026-02-22T21:36:03.698 INFO:tasks.ceph.cluster1.mgr.x.trial004.stderr:2026-02-22T21:36:03.697+0000 7f71ce7d8140 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2026-02-22T21:36:03.720 INFO:tasks.ceph.cluster1.mgr.x.trial004.stderr:2026-02-22T21:36:03.719+0000 7f71ce7d8140 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2026-02-22T21:36:03.743 INFO:tasks.ceph.cluster1.mgr.x.trial004.stderr:2026-02-22T21:36:03.742+0000 7f71ce7d8140 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2026-02-22T21:36:03.796 INFO:tasks.ceph.cluster1.mgr.x.trial004.stderr:2026-02-22T21:36:03.795+0000 7f71ce7d8140 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2026-02-22T21:36:03.817 INFO:tasks.ceph.cluster1.mgr.x.trial004.stderr:2026-02-22T21:36:03.816+0000 7f71ce7d8140 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2026-02-22T21:36:03.869 INFO:tasks.ceph.cluster1.mgr.x.trial004.stderr:2026-02-22T21:36:03.868+0000 7f71ce7d8140 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2026-02-22T21:36:04.265 INFO:tasks.ceph.cluster1.mgr.x.trial004.stderr:2026-02-22T21:36:04.263+0000 7f71ce7d8140 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2026-02-22T21:36:04.286 INFO:tasks.ceph.cluster1.mgr.x.trial004.stderr:2026-02-22T21:36:04.286+0000 7f71ce7d8140 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2026-02-22T21:36:04.319 INFO:tasks.ceph.cluster1.osd.2.trial004.stderr:2026-02-22T21:36:04.318+0000 7f28ba22c780 -1 Falling back to public interface 2026-02-22T21:36:04.627 INFO:tasks.ceph.cluster1.mgr.x.trial004.stderr:2026-02-22T21:36:04.626+0000 7f71ce7d8140 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2026-02-22T21:36:04.823 INFO:tasks.ceph.cluster1.osd.0.trial004.stderr:2026-02-22T21:36:04.821+0000 7f0a9ae0e780 -1 Falling back to public interface 2026-02-22T21:36:04.823 INFO:tasks.ceph.cluster1.osd.1.trial004.stderr:2026-02-22T21:36:04.821+0000 7f3f1c2e9780 -1 Falling back to public interface 2026-02-22T21:36:05.632 INFO:tasks.ceph.cluster1.mgr.x.trial004.stderr:2026-02-22T21:36:05.630+0000 7f70ea163640 -1 mgr.server handle_report got status from non-daemon mon.a 2026-02-22T21:36:08.322 INFO:tasks.ceph.cluster1.osd.2.trial004.stderr:2026-02-22T21:36:08.321+0000 7f28ba22c780 -1 osd.2 0 log_to_monitors true 2026-02-22T21:36:08.546 INFO:tasks.ceph.cluster1.osd.1.trial004.stderr:2026-02-22T21:36:08.545+0000 7f3f1c2e9780 -1 osd.1 0 log_to_monitors true 2026-02-22T21:36:08.601 INFO:tasks.ceph.cluster1.osd.0.trial004.stderr:2026-02-22T21:36:08.600+0000 7f0a9ae0e780 -1 osd.0 0 log_to_monitors true 2026-02-22T21:36:09.653 INFO:tasks.ceph.cluster1.osd.1.trial004.stderr:2026-02-22T21:36:09.652+0000 7f3f18276640 -1 osd.1 0 waiting for initial osdmap 2026-02-22T21:36:09.653 INFO:tasks.ceph.cluster1.osd.2.trial004.stderr:2026-02-22T21:36:09.652+0000 7f28b57dc640 -1 osd.2 0 waiting for initial osdmap 2026-02-22T21:36:09.654 INFO:tasks.ceph.cluster1.osd.0.trial004.stderr:2026-02-22T21:36:09.653+0000 7f0a967f4640 -1 osd.0 0 waiting for initial osdmap 2026-02-22T21:36:09.655 INFO:tasks.ceph.cluster1.osd.2.trial004.stderr:2026-02-22T21:36:09.654+0000 7f28ad5fe640 -1 osd.2 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2026-02-22T21:36:09.655 INFO:tasks.ceph.cluster1.osd.1.trial004.stderr:2026-02-22T21:36:09.654+0000 7f3f10098640 -1 osd.1 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2026-02-22T21:36:09.655 INFO:tasks.ceph.cluster1.osd.0.trial004.stderr:2026-02-22T21:36:09.654+0000 7f0a8e616640 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2026-02-22T21:36:09.721 DEBUG:teuthology.orchestra.run.trial004:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster1 osd dump --format=json 2026-02-22T21:36:09.874 INFO:teuthology.misc.health.trial004.stdout: 2026-02-22T21:36:09.874 INFO:teuthology.misc.health.trial004.stdout:{"epoch":7,"fsid":"54a0d3ac-2ee3-469c-a716-4416b2f462d9","created":"2026-02-22T21:36:02.327898+0000","modified":"2026-02-22T21:36:09.635026+0000","last_up_change":"0.000000","last_in_change":"2026-02-22T21:36:02.895351+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"ad69c917-7b9d-4a6d-bd05-3187fef2ebef","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"8e27aac6-4677-41a2-8575-2b92ead46c7e","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"28f6fce7-f1df-448c-91e9-b0eb54f0ff5f","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-22T21:36:09.881 DEBUG:teuthology.misc:0 of 3 OSDs are up 2026-02-22T21:36:16.184 DEBUG:teuthology.orchestra.run.trial004:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster1 osd dump --format=json 2026-02-22T21:36:16.335 INFO:teuthology.misc.health.trial004.stdout: 2026-02-22T21:36:16.335 INFO:teuthology.misc.health.trial004.stdout:{"epoch":12,"fsid":"54a0d3ac-2ee3-469c-a716-4416b2f462d9","created":"2026-02-22T21:36:02.327898+0000","modified":"2026-02-22T21:36:14.692962+0000","last_up_change":"2026-02-22T21:36:10.637268+0000","last_in_change":"2026-02-22T21:36:02.895351+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-02-22T21:36:12.642081+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ad69c917-7b9d-4a6d-bd05-3187fef2ebef","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.4:6811","nonce":157985355},{"type":"v1","addr":"10.20.193.4:6813","nonce":157985355}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.4:6815","nonce":157985355},{"type":"v1","addr":"10.20.193.4:6818","nonce":157985355}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.4:6824","nonce":157985355},{"type":"v1","addr":"10.20.193.4:6825","nonce":157985355}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.4:6821","nonce":157985355},{"type":"v1","addr":"10.20.193.4:6823","nonce":157985355}]},"public_addr":"10.20.193.4:6813/157985355","cluster_addr":"10.20.193.4:6818/157985355","heartbeat_back_addr":"10.20.193.4:6825/157985355","heartbeat_front_addr":"10.20.193.4:6823/157985355","state":["exists","up"]},{"osd":1,"uuid":"8e27aac6-4677-41a2-8575-2b92ead46c7e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":10,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.4:6810","nonce":1030571860},{"type":"v1","addr":"10.20.193.4:6812","nonce":1030571860}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.4:6814","nonce":1030571860},{"type":"v1","addr":"10.20.193.4:6816","nonce":1030571860}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.4:6820","nonce":1030571860},{"type":"v1","addr":"10.20.193.4:6822","nonce":1030571860}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.4:6817","nonce":1030571860},{"type":"v1","addr":"10.20.193.4:6819","nonce":1030571860}]},"public_addr":"10.20.193.4:6812/1030571860","cluster_addr":"10.20.193.4:6816/1030571860","heartbeat_back_addr":"10.20.193.4:6822/1030571860","heartbeat_front_addr":"10.20.193.4:6819/1030571860","state":["exists","up"]},{"osd":2,"uuid":"28f6fce7-f1df-448c-91e9-b0eb54f0ff5f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.4:6800","nonce":283052529},{"type":"v1","addr":"10.20.193.4:6801","nonce":283052529}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.4:6802","nonce":283052529},{"type":"v1","addr":"10.20.193.4:6803","nonce":283052529}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.4:6806","nonce":283052529},{"type":"v1","addr":"10.20.193.4:6807","nonce":283052529}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.4:6804","nonce":283052529},{"type":"v1","addr":"10.20.193.4:6805","nonce":283052529}]},"public_addr":"10.20.193.4:6801/283052529","cluster_addr":"10.20.193.4:6803/283052529","heartbeat_back_addr":"10.20.193.4:6807/283052529","heartbeat_front_addr":"10.20.193.4:6805/283052529","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T21:36:09.625984+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T21:36:09.570742+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T21:36:09.368412+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-22T21:36:16.343 DEBUG:teuthology.misc:3 of 3 OSDs are up 2026-02-22T21:36:16.343 INFO:tasks.ceph:Creating RBD pool 2026-02-22T21:36:16.343 DEBUG:teuthology.orchestra.run.trial004:> sudo ceph --cluster cluster1 osd pool create rbd 8 2026-02-22T21:36:16.699 INFO:teuthology.orchestra.run.trial004.stderr:pool 'rbd' created 2026-02-22T21:36:16.710 DEBUG:teuthology.orchestra.run.trial004:> rbd --cluster cluster1 pool init rbd 2026-02-22T21:36:19.711 INFO:tasks.ceph:Starting mds daemons in cluster cluster1... 2026-02-22T21:36:19.711 INFO:tasks.ceph:Waiting until cluster1 daemons up and pgs clean... 2026-02-22T21:36:19.711 INFO:tasks.ceph.ceph_manager.cluster1:waiting for mgr available 2026-02-22T21:36:19.711 DEBUG:teuthology.orchestra.run.trial004:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 mgr dump --format=json 2026-02-22T21:36:19.711 INFO:tasks.daemonwatchdog.daemon_watchdog:watchdog starting 2026-02-22T21:36:19.888 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:36:19.895 INFO:teuthology.orchestra.run.trial004.stdout:{"epoch":5,"flags":0,"active_gid":4102,"active_name":"x","active_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.4:6808","nonce":777226972},{"type":"v1","addr":"10.20.193.4:6809","nonce":777226972}]},"active_addr":"10.20.193.4:6809/777226972","active_change":"2026-02-22T21:36:04.628021+0000","active_mgr_features":4540701547738038271,"available":true,"standbys":[],"modules":["iostat","nfs","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"update_pg_upmap_activity":{"name":"update_pg_upmap_activity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Updates pg_upmap activity stats to be used in `balancer status detail`","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:10.4.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.7.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.2.5","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.51.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba":{"name":"container_image_samba","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-server:devbuilds-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba/SMB container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"quay.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"location of dashboards to include in grafana deployments","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"ssh_keepalive_count_max":{"name":"ssh_keepalive_count_max","type":"int","level":"advanced","flags":0,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"How many times ssh connections can fail liveness checks before the host is marked offline","long_desc":"","tags":[],"see_also":[]},"ssh_keepalive_interval":{"name":"ssh_keepalive_interval","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"How often ssh connections are checked for liveness","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":true,"error_string":"","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_requests":{"name":"max_requests","type":"int","level":"advanced","flags":0,"default_value":"500","min":"","max":"","enum_allowed":[],"desc":"Maximum number of requests to keep in memory. When new request comes in, the oldest request will be removed if the number of requests exceeds the max request number. if un-finished request is removed, error message will be logged in the ceph-mgr log.","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"secondary_zone_period_retry_limit":{"name":"secondary_zone_period_retry_limit","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"RGW module period update retry limit for secondary site","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"squid":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"force_disabled_modules":{},"last_failure_osd_epoch":0,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"10.20.193.4:0","nonce":1465589420}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"10.20.193.4:0","nonce":1074930875}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"10.20.193.4:0","nonce":934557135}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"10.20.193.4:0","nonce":2576623318}]}]} 2026-02-22T21:36:19.899 INFO:tasks.ceph.ceph_manager.cluster1:mgr available! 2026-02-22T21:36:19.899 INFO:tasks.ceph.ceph_manager.cluster1:waiting for all up 2026-02-22T21:36:19.900 DEBUG:teuthology.orchestra.run.trial004:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd dump --format=json 2026-02-22T21:36:20.053 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:36:20.053 INFO:teuthology.orchestra.run.trial004.stdout:{"epoch":16,"fsid":"54a0d3ac-2ee3-469c-a716-4416b2f462d9","created":"2026-02-22T21:36:02.327898+0000","modified":"2026-02-22T21:36:19.702026+0000","last_up_change":"2026-02-22T21:36:10.637268+0000","last_in_change":"2026-02-22T21:36:02.895351+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-02-22T21:36:12.642081+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2026-02-22T21:36:16.499800+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":16,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ad69c917-7b9d-4a6d-bd05-3187fef2ebef","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":13,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.4:6811","nonce":157985355},{"type":"v1","addr":"10.20.193.4:6813","nonce":157985355}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.4:6815","nonce":157985355},{"type":"v1","addr":"10.20.193.4:6818","nonce":157985355}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.4:6824","nonce":157985355},{"type":"v1","addr":"10.20.193.4:6825","nonce":157985355}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.4:6821","nonce":157985355},{"type":"v1","addr":"10.20.193.4:6823","nonce":157985355}]},"public_addr":"10.20.193.4:6813/157985355","cluster_addr":"10.20.193.4:6818/157985355","heartbeat_back_addr":"10.20.193.4:6825/157985355","heartbeat_front_addr":"10.20.193.4:6823/157985355","state":["exists","up"]},{"osd":1,"uuid":"8e27aac6-4677-41a2-8575-2b92ead46c7e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":13,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.4:6810","nonce":1030571860},{"type":"v1","addr":"10.20.193.4:6812","nonce":1030571860}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.4:6814","nonce":1030571860},{"type":"v1","addr":"10.20.193.4:6816","nonce":1030571860}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.4:6820","nonce":1030571860},{"type":"v1","addr":"10.20.193.4:6822","nonce":1030571860}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.4:6817","nonce":1030571860},{"type":"v1","addr":"10.20.193.4:6819","nonce":1030571860}]},"public_addr":"10.20.193.4:6812/1030571860","cluster_addr":"10.20.193.4:6816/1030571860","heartbeat_back_addr":"10.20.193.4:6822/1030571860","heartbeat_front_addr":"10.20.193.4:6819/1030571860","state":["exists","up"]},{"osd":2,"uuid":"28f6fce7-f1df-448c-91e9-b0eb54f0ff5f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":13,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.4:6800","nonce":283052529},{"type":"v1","addr":"10.20.193.4:6801","nonce":283052529}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.4:6802","nonce":283052529},{"type":"v1","addr":"10.20.193.4:6803","nonce":283052529}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.4:6806","nonce":283052529},{"type":"v1","addr":"10.20.193.4:6807","nonce":283052529}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.4:6804","nonce":283052529},{"type":"v1","addr":"10.20.193.4:6805","nonce":283052529}]},"public_addr":"10.20.193.4:6801/283052529","cluster_addr":"10.20.193.4:6803/283052529","heartbeat_back_addr":"10.20.193.4:6807/283052529","heartbeat_front_addr":"10.20.193.4:6805/283052529","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T21:36:09.625984+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T21:36:09.570742+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T21:36:09.368412+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-22T21:36:20.062 INFO:tasks.ceph.ceph_manager.cluster1:all up! 2026-02-22T21:36:20.062 DEBUG:teuthology.orchestra.run.trial004:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd dump --format=json 2026-02-22T21:36:20.218 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:36:20.218 INFO:teuthology.orchestra.run.trial004.stdout:{"epoch":16,"fsid":"54a0d3ac-2ee3-469c-a716-4416b2f462d9","created":"2026-02-22T21:36:02.327898+0000","modified":"2026-02-22T21:36:19.702026+0000","last_up_change":"2026-02-22T21:36:10.637268+0000","last_in_change":"2026-02-22T21:36:02.895351+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-02-22T21:36:12.642081+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2026-02-22T21:36:16.499800+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":16,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ad69c917-7b9d-4a6d-bd05-3187fef2ebef","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":13,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.4:6811","nonce":157985355},{"type":"v1","addr":"10.20.193.4:6813","nonce":157985355}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.4:6815","nonce":157985355},{"type":"v1","addr":"10.20.193.4:6818","nonce":157985355}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.4:6824","nonce":157985355},{"type":"v1","addr":"10.20.193.4:6825","nonce":157985355}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.4:6821","nonce":157985355},{"type":"v1","addr":"10.20.193.4:6823","nonce":157985355}]},"public_addr":"10.20.193.4:6813/157985355","cluster_addr":"10.20.193.4:6818/157985355","heartbeat_back_addr":"10.20.193.4:6825/157985355","heartbeat_front_addr":"10.20.193.4:6823/157985355","state":["exists","up"]},{"osd":1,"uuid":"8e27aac6-4677-41a2-8575-2b92ead46c7e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":13,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.4:6810","nonce":1030571860},{"type":"v1","addr":"10.20.193.4:6812","nonce":1030571860}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.4:6814","nonce":1030571860},{"type":"v1","addr":"10.20.193.4:6816","nonce":1030571860}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.4:6820","nonce":1030571860},{"type":"v1","addr":"10.20.193.4:6822","nonce":1030571860}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.4:6817","nonce":1030571860},{"type":"v1","addr":"10.20.193.4:6819","nonce":1030571860}]},"public_addr":"10.20.193.4:6812/1030571860","cluster_addr":"10.20.193.4:6816/1030571860","heartbeat_back_addr":"10.20.193.4:6822/1030571860","heartbeat_front_addr":"10.20.193.4:6819/1030571860","state":["exists","up"]},{"osd":2,"uuid":"28f6fce7-f1df-448c-91e9-b0eb54f0ff5f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":13,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.4:6800","nonce":283052529},{"type":"v1","addr":"10.20.193.4:6801","nonce":283052529}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.4:6802","nonce":283052529},{"type":"v1","addr":"10.20.193.4:6803","nonce":283052529}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.4:6806","nonce":283052529},{"type":"v1","addr":"10.20.193.4:6807","nonce":283052529}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.4:6804","nonce":283052529},{"type":"v1","addr":"10.20.193.4:6805","nonce":283052529}]},"public_addr":"10.20.193.4:6801/283052529","cluster_addr":"10.20.193.4:6803/283052529","heartbeat_back_addr":"10.20.193.4:6807/283052529","heartbeat_front_addr":"10.20.193.4:6805/283052529","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T21:36:09.625984+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T21:36:09.570742+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T21:36:09.368412+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-22T21:36:20.226 DEBUG:teuthology.orchestra.run.trial004:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.0 flush_pg_stats 2026-02-22T21:36:20.226 DEBUG:teuthology.orchestra.run.trial004:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.1 flush_pg_stats 2026-02-22T21:36:20.226 DEBUG:teuthology.orchestra.run.trial004:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.2 flush_pg_stats 2026-02-22T21:36:20.312 INFO:teuthology.orchestra.run.trial004.stdout:34359738372 2026-02-22T21:36:20.312 DEBUG:teuthology.orchestra.run.trial004:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.2 2026-02-22T21:36:20.314 INFO:teuthology.orchestra.run.trial004.stdout:34359738372 2026-02-22T21:36:20.314 DEBUG:teuthology.orchestra.run.trial004:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.0 2026-02-22T21:36:20.315 INFO:teuthology.orchestra.run.trial004.stdout:34359738372 2026-02-22T21:36:20.316 DEBUG:teuthology.orchestra.run.trial004:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.1 2026-02-22T21:36:20.514 INFO:teuthology.orchestra.run.trial004.stdout:34359738371 2026-02-22T21:36:20.516 INFO:teuthology.orchestra.run.trial004.stdout:34359738371 2026-02-22T21:36:20.518 INFO:teuthology.orchestra.run.trial004.stdout:34359738371 2026-02-22T21:36:20.524 INFO:tasks.ceph.ceph_manager.cluster1:need seq 34359738372 got 34359738371 for osd.2 2026-02-22T21:36:20.525 INFO:tasks.ceph.ceph_manager.cluster1:need seq 34359738372 got 34359738371 for osd.0 2026-02-22T21:36:20.528 INFO:tasks.ceph.ceph_manager.cluster1:need seq 34359738372 got 34359738371 for osd.1 2026-02-22T21:36:21.525 DEBUG:teuthology.orchestra.run.trial004:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.2 2026-02-22T21:36:21.526 DEBUG:teuthology.orchestra.run.trial004:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.0 2026-02-22T21:36:21.528 DEBUG:teuthology.orchestra.run.trial004:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.1 2026-02-22T21:36:21.685 INFO:teuthology.orchestra.run.trial004.stdout:34359738372 2026-02-22T21:36:21.687 INFO:teuthology.orchestra.run.trial004.stdout:34359738372 2026-02-22T21:36:21.695 INFO:tasks.ceph.ceph_manager.cluster1:need seq 34359738372 got 34359738372 for osd.0 2026-02-22T21:36:21.696 DEBUG:teuthology.parallel:result is None 2026-02-22T21:36:21.697 INFO:tasks.ceph.ceph_manager.cluster1:need seq 34359738372 got 34359738372 for osd.2 2026-02-22T21:36:21.697 DEBUG:teuthology.parallel:result is None 2026-02-22T21:36:21.727 INFO:teuthology.orchestra.run.trial004.stdout:34359738372 2026-02-22T21:36:21.736 INFO:tasks.ceph.ceph_manager.cluster1:need seq 34359738372 got 34359738372 for osd.1 2026-02-22T21:36:21.737 DEBUG:teuthology.parallel:result is None 2026-02-22T21:36:21.737 INFO:tasks.ceph.ceph_manager.cluster1:waiting for clean 2026-02-22T21:36:21.737 DEBUG:teuthology.orchestra.run.trial004:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2026-02-22T21:36:21.931 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:36:21.931 INFO:teuthology.orchestra.run.trial004.stderr:dumped all 2026-02-22T21:36:21.938 INFO:teuthology.orchestra.run.trial004.stdout:{"pg_ready":true,"pg_map":{"version":19,"stamp":"2026-02-22T21:36:20.631771+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":17,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":81412,"kb_used_data":512,"kb_used_omap":4,"kb_used_meta":80827,"kb_avail":283034108,"statfs":{"total":289910292480,"available":289826926592,"internally_reserved":0,"allocated":524288,"data_stored":1295954,"data_compressed":19658,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":4782,"internal_metadata":82767186},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"3.932831"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T21:36:19.704598+0000","last_change":"2026-02-22T21:36:19.704598+0000","last_active":"2026-02-22T21:36:19.704598+0000","last_peered":"2026-02-22T21:36:19.704598+0000","last_clean":"2026-02-22T21:36:19.704598+0000","last_became_active":"2026-02-22T21:36:17.702763+0000","last_became_peered":"2026-02-22T21:36:17.702763+0000","last_unstale":"2026-02-22T21:36:19.704598+0000","last_undegraded":"2026-02-22T21:36:19.704598+0000","last_fullsized":"2026-02-22T21:36:19.704598+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:16.695655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:16.695655+0000","last_clean_scrub_stamp":"2026-02-22T21:36:16.695655+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T21:59:04.606522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00062915900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.6","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T21:36:19.704496+0000","last_change":"2026-02-22T21:36:19.704496+0000","last_active":"2026-02-22T21:36:19.704496+0000","last_peered":"2026-02-22T21:36:19.704496+0000","last_clean":"2026-02-22T21:36:19.704496+0000","last_became_active":"2026-02-22T21:36:17.703106+0000","last_became_peered":"2026-02-22T21:36:17.703106+0000","last_unstale":"2026-02-22T21:36:19.704496+0000","last_undegraded":"2026-02-22T21:36:19.704496+0000","last_fullsized":"2026-02-22T21:36:19.704496+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:16.695655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:16.695655+0000","last_clean_scrub_stamp":"2026-02-22T21:36:16.695655+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T03:56:26.669952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00053011499999999995,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.5","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T21:36:19.704255+0000","last_change":"2026-02-22T21:36:19.704255+0000","last_active":"2026-02-22T21:36:19.704255+0000","last_peered":"2026-02-22T21:36:19.704255+0000","last_clean":"2026-02-22T21:36:19.704255+0000","last_became_active":"2026-02-22T21:36:17.702758+0000","last_became_peered":"2026-02-22T21:36:17.702758+0000","last_unstale":"2026-02-22T21:36:19.704255+0000","last_undegraded":"2026-02-22T21:36:19.704255+0000","last_fullsized":"2026-02-22T21:36:19.704255+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:16.695655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:16.695655+0000","last_clean_scrub_stamp":"2026-02-22T21:36:16.695655+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T01:00:00.761061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033481199999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.4","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T21:36:19.704343+0000","last_change":"2026-02-22T21:36:19.704343+0000","last_active":"2026-02-22T21:36:19.704343+0000","last_peered":"2026-02-22T21:36:19.704343+0000","last_clean":"2026-02-22T21:36:19.704343+0000","last_became_active":"2026-02-22T21:36:17.702784+0000","last_became_peered":"2026-02-22T21:36:17.702784+0000","last_unstale":"2026-02-22T21:36:19.704343+0000","last_undegraded":"2026-02-22T21:36:19.704343+0000","last_fullsized":"2026-02-22T21:36:19.704343+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:16.695655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:16.695655+0000","last_clean_scrub_stamp":"2026-02-22T21:36:16.695655+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T01:51:15.121856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000400474,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.2","version":"16'2","reported_seq":21,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T21:36:19.706811+0000","last_change":"2026-02-22T21:36:19.706373+0000","last_active":"2026-02-22T21:36:19.706811+0000","last_peered":"2026-02-22T21:36:19.706811+0000","last_clean":"2026-02-22T21:36:19.706811+0000","last_became_active":"2026-02-22T21:36:17.702836+0000","last_became_peered":"2026-02-22T21:36:17.702836+0000","last_unstale":"2026-02-22T21:36:19.706811+0000","last_undegraded":"2026-02-22T21:36:19.706811+0000","last_fullsized":"2026-02-22T21:36:19.706811+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:16.695655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:16.695655+0000","last_clean_scrub_stamp":"2026-02-22T21:36:16.695655+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T09:04:07.404872+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00089484299999999998,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.1","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T21:36:19.749070+0000","last_change":"2026-02-22T21:36:19.749070+0000","last_active":"2026-02-22T21:36:19.749070+0000","last_peered":"2026-02-22T21:36:19.749070+0000","last_clean":"2026-02-22T21:36:19.749070+0000","last_became_active":"2026-02-22T21:36:17.702842+0000","last_became_peered":"2026-02-22T21:36:17.702842+0000","last_unstale":"2026-02-22T21:36:19.749070+0000","last_undegraded":"2026-02-22T21:36:19.749070+0000","last_fullsized":"2026-02-22T21:36:19.749070+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:16.695655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:16.695655+0000","last_clean_scrub_stamp":"2026-02-22T21:36:16.695655+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T08:47:34.298671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00023738200000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.0","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T21:36:19.749053+0000","last_change":"2026-02-22T21:36:19.749053+0000","last_active":"2026-02-22T21:36:19.749053+0000","last_peered":"2026-02-22T21:36:19.749053+0000","last_clean":"2026-02-22T21:36:19.749053+0000","last_became_active":"2026-02-22T21:36:17.702618+0000","last_became_peered":"2026-02-22T21:36:17.702618+0000","last_unstale":"2026-02-22T21:36:19.749053+0000","last_undegraded":"2026-02-22T21:36:19.749053+0000","last_fullsized":"2026-02-22T21:36:19.749053+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:16.695655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:16.695655+0000","last_clean_scrub_stamp":"2026-02-22T21:36:16.695655+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T23:00:17.618828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026033399999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.3","version":"14'1","reported_seq":20,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T21:36:19.704451+0000","last_change":"2026-02-22T21:36:19.704451+0000","last_active":"2026-02-22T21:36:19.704451+0000","last_peered":"2026-02-22T21:36:19.704451+0000","last_clean":"2026-02-22T21:36:19.704451+0000","last_became_active":"2026-02-22T21:36:17.702257+0000","last_became_peered":"2026-02-22T21:36:17.702257+0000","last_unstale":"2026-02-22T21:36:19.704451+0000","last_undegraded":"2026-02-22T21:36:19.704451+0000","last_fullsized":"2026-02-22T21:36:19.704451+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:16.695655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:16.695655+0000","last_clean_scrub_stamp":"2026-02-22T21:36:16.695655+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T01:42:15.152043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00049669299999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"1.0","version":"11'72","reported_seq":110,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T21:36:19.703808+0000","last_change":"2026-02-22T21:36:14.450229+0000","last_active":"2026-02-22T21:36:19.703808+0000","last_peered":"2026-02-22T21:36:19.703808+0000","last_clean":"2026-02-22T21:36:19.703808+0000","last_became_active":"2026-02-22T21:36:14.449500+0000","last_became_peered":"2026-02-22T21:36:14.449500+0000","last_unstale":"2026-02-22T21:36:19.703808+0000","last_undegraded":"2026-02-22T21:36:19.703808+0000","last_fullsized":"2026-02-22T21:36:19.703808+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:12.692166+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:12.692166+0000","last_clean_scrub_stamp":"2026-02-22T21:36:12.692166+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T02:36:42.881463+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":1180736,"data_compressed":19658,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738372,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27108,"kb_used_data":136,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94344732,"statfs":{"total":96636764160,"available":96609005568,"internally_reserved":0,"allocated":139264,"data_stored":40774,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27589062},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738372,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27152,"kb_used_data":188,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94344688,"statfs":{"total":96636764160,"available":96608960512,"internally_reserved":0,"allocated":192512,"data_stored":627590,"data_compressed":9829,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1594,"internal_metadata":27589062},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738372,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27152,"kb_used_data":188,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94344688,"statfs":{"total":96636764160,"available":96608960512,"internally_reserved":0,"allocated":192512,"data_stored":627590,"data_compressed":9829,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1594,"internal_metadata":27589062},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":9829,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":9829,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-22T21:36:21.939 DEBUG:teuthology.orchestra.run.trial004:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2026-02-22T21:36:22.094 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:36:22.095 INFO:teuthology.orchestra.run.trial004.stderr:dumped all 2026-02-22T21:36:22.102 INFO:teuthology.orchestra.run.trial004.stdout:{"pg_ready":true,"pg_map":{"version":19,"stamp":"2026-02-22T21:36:20.631771+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":17,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":81412,"kb_used_data":512,"kb_used_omap":4,"kb_used_meta":80827,"kb_avail":283034108,"statfs":{"total":289910292480,"available":289826926592,"internally_reserved":0,"allocated":524288,"data_stored":1295954,"data_compressed":19658,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":4782,"internal_metadata":82767186},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"3.932831"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T21:36:19.704598+0000","last_change":"2026-02-22T21:36:19.704598+0000","last_active":"2026-02-22T21:36:19.704598+0000","last_peered":"2026-02-22T21:36:19.704598+0000","last_clean":"2026-02-22T21:36:19.704598+0000","last_became_active":"2026-02-22T21:36:17.702763+0000","last_became_peered":"2026-02-22T21:36:17.702763+0000","last_unstale":"2026-02-22T21:36:19.704598+0000","last_undegraded":"2026-02-22T21:36:19.704598+0000","last_fullsized":"2026-02-22T21:36:19.704598+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:16.695655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:16.695655+0000","last_clean_scrub_stamp":"2026-02-22T21:36:16.695655+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T21:59:04.606522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00062915900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.6","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T21:36:19.704496+0000","last_change":"2026-02-22T21:36:19.704496+0000","last_active":"2026-02-22T21:36:19.704496+0000","last_peered":"2026-02-22T21:36:19.704496+0000","last_clean":"2026-02-22T21:36:19.704496+0000","last_became_active":"2026-02-22T21:36:17.703106+0000","last_became_peered":"2026-02-22T21:36:17.703106+0000","last_unstale":"2026-02-22T21:36:19.704496+0000","last_undegraded":"2026-02-22T21:36:19.704496+0000","last_fullsized":"2026-02-22T21:36:19.704496+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:16.695655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:16.695655+0000","last_clean_scrub_stamp":"2026-02-22T21:36:16.695655+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T03:56:26.669952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00053011499999999995,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.5","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T21:36:19.704255+0000","last_change":"2026-02-22T21:36:19.704255+0000","last_active":"2026-02-22T21:36:19.704255+0000","last_peered":"2026-02-22T21:36:19.704255+0000","last_clean":"2026-02-22T21:36:19.704255+0000","last_became_active":"2026-02-22T21:36:17.702758+0000","last_became_peered":"2026-02-22T21:36:17.702758+0000","last_unstale":"2026-02-22T21:36:19.704255+0000","last_undegraded":"2026-02-22T21:36:19.704255+0000","last_fullsized":"2026-02-22T21:36:19.704255+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:16.695655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:16.695655+0000","last_clean_scrub_stamp":"2026-02-22T21:36:16.695655+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T01:00:00.761061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033481199999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.4","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T21:36:19.704343+0000","last_change":"2026-02-22T21:36:19.704343+0000","last_active":"2026-02-22T21:36:19.704343+0000","last_peered":"2026-02-22T21:36:19.704343+0000","last_clean":"2026-02-22T21:36:19.704343+0000","last_became_active":"2026-02-22T21:36:17.702784+0000","last_became_peered":"2026-02-22T21:36:17.702784+0000","last_unstale":"2026-02-22T21:36:19.704343+0000","last_undegraded":"2026-02-22T21:36:19.704343+0000","last_fullsized":"2026-02-22T21:36:19.704343+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:16.695655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:16.695655+0000","last_clean_scrub_stamp":"2026-02-22T21:36:16.695655+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T01:51:15.121856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000400474,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.2","version":"16'2","reported_seq":21,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T21:36:19.706811+0000","last_change":"2026-02-22T21:36:19.706373+0000","last_active":"2026-02-22T21:36:19.706811+0000","last_peered":"2026-02-22T21:36:19.706811+0000","last_clean":"2026-02-22T21:36:19.706811+0000","last_became_active":"2026-02-22T21:36:17.702836+0000","last_became_peered":"2026-02-22T21:36:17.702836+0000","last_unstale":"2026-02-22T21:36:19.706811+0000","last_undegraded":"2026-02-22T21:36:19.706811+0000","last_fullsized":"2026-02-22T21:36:19.706811+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:16.695655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:16.695655+0000","last_clean_scrub_stamp":"2026-02-22T21:36:16.695655+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T09:04:07.404872+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00089484299999999998,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.1","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T21:36:19.749070+0000","last_change":"2026-02-22T21:36:19.749070+0000","last_active":"2026-02-22T21:36:19.749070+0000","last_peered":"2026-02-22T21:36:19.749070+0000","last_clean":"2026-02-22T21:36:19.749070+0000","last_became_active":"2026-02-22T21:36:17.702842+0000","last_became_peered":"2026-02-22T21:36:17.702842+0000","last_unstale":"2026-02-22T21:36:19.749070+0000","last_undegraded":"2026-02-22T21:36:19.749070+0000","last_fullsized":"2026-02-22T21:36:19.749070+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:16.695655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:16.695655+0000","last_clean_scrub_stamp":"2026-02-22T21:36:16.695655+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T08:47:34.298671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00023738200000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.0","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T21:36:19.749053+0000","last_change":"2026-02-22T21:36:19.749053+0000","last_active":"2026-02-22T21:36:19.749053+0000","last_peered":"2026-02-22T21:36:19.749053+0000","last_clean":"2026-02-22T21:36:19.749053+0000","last_became_active":"2026-02-22T21:36:17.702618+0000","last_became_peered":"2026-02-22T21:36:17.702618+0000","last_unstale":"2026-02-22T21:36:19.749053+0000","last_undegraded":"2026-02-22T21:36:19.749053+0000","last_fullsized":"2026-02-22T21:36:19.749053+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:16.695655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:16.695655+0000","last_clean_scrub_stamp":"2026-02-22T21:36:16.695655+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T23:00:17.618828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026033399999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.3","version":"14'1","reported_seq":20,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T21:36:19.704451+0000","last_change":"2026-02-22T21:36:19.704451+0000","last_active":"2026-02-22T21:36:19.704451+0000","last_peered":"2026-02-22T21:36:19.704451+0000","last_clean":"2026-02-22T21:36:19.704451+0000","last_became_active":"2026-02-22T21:36:17.702257+0000","last_became_peered":"2026-02-22T21:36:17.702257+0000","last_unstale":"2026-02-22T21:36:19.704451+0000","last_undegraded":"2026-02-22T21:36:19.704451+0000","last_fullsized":"2026-02-22T21:36:19.704451+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:16.695655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:16.695655+0000","last_clean_scrub_stamp":"2026-02-22T21:36:16.695655+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T01:42:15.152043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00049669299999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"1.0","version":"11'72","reported_seq":110,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T21:36:19.703808+0000","last_change":"2026-02-22T21:36:14.450229+0000","last_active":"2026-02-22T21:36:19.703808+0000","last_peered":"2026-02-22T21:36:19.703808+0000","last_clean":"2026-02-22T21:36:19.703808+0000","last_became_active":"2026-02-22T21:36:14.449500+0000","last_became_peered":"2026-02-22T21:36:14.449500+0000","last_unstale":"2026-02-22T21:36:19.703808+0000","last_undegraded":"2026-02-22T21:36:19.703808+0000","last_fullsized":"2026-02-22T21:36:19.703808+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:12.692166+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:12.692166+0000","last_clean_scrub_stamp":"2026-02-22T21:36:12.692166+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T02:36:42.881463+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":1180736,"data_compressed":19658,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738372,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27108,"kb_used_data":136,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94344732,"statfs":{"total":96636764160,"available":96609005568,"internally_reserved":0,"allocated":139264,"data_stored":40774,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27589062},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738372,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27152,"kb_used_data":188,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94344688,"statfs":{"total":96636764160,"available":96608960512,"internally_reserved":0,"allocated":192512,"data_stored":627590,"data_compressed":9829,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1594,"internal_metadata":27589062},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738372,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27152,"kb_used_data":188,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94344688,"statfs":{"total":96636764160,"available":96608960512,"internally_reserved":0,"allocated":192512,"data_stored":627590,"data_compressed":9829,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1594,"internal_metadata":27589062},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":9829,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":9829,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-22T21:36:22.104 INFO:tasks.ceph.ceph_manager.cluster1:clean! 2026-02-22T21:36:22.104 INFO:tasks.ceph:Waiting until ceph cluster cluster1 is healthy... 2026-02-22T21:36:22.104 INFO:tasks.ceph.ceph_manager.cluster1:wait_until_healthy 2026-02-22T21:36:22.104 DEBUG:teuthology.orchestra.run.trial004:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 health --format=json 2026-02-22T21:36:22.276 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:36:22.277 INFO:teuthology.orchestra.run.trial004.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2026-02-22T21:36:22.285 INFO:tasks.ceph.ceph_manager.cluster1:wait_until_healthy done 2026-02-22T21:36:22.285 INFO:teuthology.run_tasks:Running task ceph... 2026-02-22T21:36:22.296 INFO:tasks.ceph:Creating ceph cluster cluster2... 2026-02-22T21:36:22.296 INFO:tasks.ceph:config {'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'simple'}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bluestore block size': 96636764160, 'bluestore compression algorithm': 'lz4', 'bluestore compression mode': 'aggressive', 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'fs': 'xfs', 'mkfs_options': None, 'mount_options': None, 'skip_mgr_daemons': False, 'log_ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'cpu_profile': set(), 'cluster': 'cluster2', 'mon_bind_msgr2': True, 'mon_bind_addrvec': True} 2026-02-22T21:36:22.296 INFO:tasks.ceph:ctx.config {'archive_path': '/home/teuthworker/mnt/teuthology/yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial/64060', 'branch': 'wip-yuri6-testing-2026-02-22-1629-squid', 'description': 'rbd/mirror-thrash/{base/install clients/mirror cluster/{2-node openstack} conf/{disable-pool-app} msgr-failures/few objectstore/bluestore-comp-lz4 policy/simple rbd-mirror/four-per-cluster supported-random-distro$/{centos_latest} workloads/rbd-mirror-snapshot-stress-workunit-exclusive-lock}', 'email': 'yweinste@redhat.com', 'first_in_suite': False, 'flavor': 'default', 'job_id': '64060', 'kernel': {'branch': 'distro', 'kdb': True, 'sha1': 'distro'}, 'last_in_suite': False, 'machine_type': 'trial', 'meta': [{'desc': 'run two ceph clusters and install rbd-mirror'}, {'desc': 'configure the permissions for client.mirror'}, {'desc': '2 ceph clusters with 1 mon and 3 osds each'}, {'desc': 'run four rbd-mirror daemons per cluster'}, {'desc': 'run the rbd_mirror_stress.sh workunit to test the rbd-mirror daemon'}], 'name': 'yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial', 'no_nested_subset': False, 'openstack': [{'volumes': {'count': 3, 'size': 30}}], 'os_type': 'centos', 'os_version': '9.stream', 'overrides': {'admin_socket': {'branch': 'wip-yuri6-testing-2026-02-22-1629-squid'}, 'ceph': {'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'simple'}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bluestore block size': 96636764160, 'bluestore compression algorithm': 'lz4', 'bluestore compression mode': 'aggressive', 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579'}, 'ceph-deploy': {'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {}}}, 'install': {'ceph': {'flavor': 'default', 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579'}}, 'selinux': {'allowlist': ['scontext=system_u:system_r:getty_t:s0']}, 'thrashosds': {'bdev_inject_crash': 2, 'bdev_inject_crash_probability': 0.5}, 'workunit': {'branch': 'wip-yuri6-testing-2026-02-22-1629-squid', 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579'}}, 'owner': 'scheduled_yuriw@soko04.front.sepia.ceph.com', 'priority': 69, 'repo': 'https://github.com/ceph/ceph-ci.git', 'roles': [['cluster1.mon.a', 'cluster1.mgr.x', 'cluster2.mgr.x', 'cluster1.osd.0', 'cluster1.osd.1', 'cluster1.osd.2', 'cluster1.client.0', 'cluster2.client.0'], ['cluster2.mon.a', 'cluster2.osd.0', 'cluster2.osd.1', 'cluster2.osd.2', 'cluster1.client.mirror', 'cluster1.client.mirror.0', 'cluster1.client.mirror.1', 'cluster1.client.mirror.2', 'cluster1.client.mirror.3', 'cluster1.client.mirror.4', 'cluster1.client.mirror.5', 'cluster1.client.mirror.6', 'cluster2.client.mirror', 'cluster2.client.mirror.0', 'cluster2.client.mirror.1', 'cluster2.client.mirror.2', 'cluster2.client.mirror.3', 'cluster2.client.mirror.4', 'cluster2.client.mirror.5', 'cluster2.client.mirror.6']], 'seed': 9196, 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579', 'sleep_before_teardown': 0, 'subset': '111/120000', 'suite': 'rbd', 'suite_branch': 'wip-yuri6-testing-2026-02-22-1629-squid', 'suite_path': '/home/teuthworker/src/github.com_ceph_ceph-c_f90f4eaebb28b619cf76e80cbbe997656da69579/qa', 'suite_relpath': 'qa', 'suite_repo': 'https://github.com/ceph/ceph-ci.git', 'suite_sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579', 'targets': {'trial004.front.sepia.ceph.com': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBDkPjQYprX4CF30+jAbTd9L3HjPvsP5GsEYCy0m1NpRHUc7PIKylXMo7Pk3bw2gAMNlmx9b6E/JJW/eTIjrHUGg=', 'trial026.front.sepia.ceph.com': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBM5AXTZqVQNBK8dSoTzQSzjoGT4p0yrUpjtrv9BGY3wKuaRfx3V9PHQ5dxCaBeyOqt+DwHWF93iRGCUbLiIW/wQ='}, 'tasks': [{'internal.check_packages': None}, {'internal.buildpackages_prep': None}, {'internal.save_config': None}, {'internal.check_lock': None}, {'internal.add_remotes': None}, {'console_log': None}, {'internal.connect': None}, {'internal.push_inventory': None}, {'internal.serialize_remote_roles': None}, {'internal.check_conflict': None}, {'internal.check_ceph_data': None}, {'internal.vm_setup': None}, {'kernel': {'branch': 'distro', 'kdb': True, 'sha1': 'distro'}}, {'internal.base': None}, {'internal.archive_upload': None}, {'internal.archive': None}, {'internal.coredump': None}, {'internal.sudo': None}, {'internal.syslog': None}, {'internal.timer': None}, {'pcp': None}, {'selinux': None}, {'ansible.cephlab': None}, {'clock': None}, {'install': {'extra_packages': ['rbd-mirror'], 'flavor': 'default', 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579'}}, {'ceph': {'cluster': 'cluster1', 'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'simple'}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bluestore block size': 96636764160, 'bluestore compression algorithm': 'lz4', 'bluestore compression mode': 'aggressive', 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579'}}, {'ceph': {'cluster': 'cluster2', 'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'simple'}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bluestore block size': 96636764160, 'bluestore compression algorithm': 'lz4', 'bluestore compression mode': 'aggressive', 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579'}}, {'exec': {'cluster1.client.mirror.0': ["sudo ceph --cluster cluster1 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd'"]}}, {'rbd-mirror': {'client': 'cluster1.client.mirror.0', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster1.client.mirror.1', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster1.client.mirror.2', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster1.client.mirror.3', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster2.client.mirror.0', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster2.client.mirror.1', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster2.client.mirror.2', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster2.client.mirror.3', 'thrash': True}}, {'rbd-mirror-thrash': {'cluster': 'cluster1'}}, {'rbd-mirror-thrash': {'cluster': 'cluster2'}}, {'workunit': {'clients': {'cluster1.client.mirror': ['rbd/rbd_mirror_stress.sh']}, 'env': {'CEPH_ARGS': '', 'MIRROR_POOL_MODE': 'image', 'RBD_IMAGE_FEATURES': 'layering,exclusive-lock', 'RBD_MIRROR_INSTANCES': '4', 'RBD_MIRROR_MODE': 'snapshot', 'RBD_MIRROR_USE_EXISTING_CLUSTER': '1', 'RBD_MIRROR_USE_RBD_MIRROR': '1'}, 'timeout': '6h'}}], 'teuthology': {'fragments_dropped': [], 'meta': {}, 'postmerge': []}, 'teuthology_branch': 'main', 'teuthology_sha1': 'fa17720d0088c3ac28e473468bfc79eeaff5cd38', 'timestamp': '2026-02-22_19:35:38', 'tube': 'trial', 'user': 'yuriw', 'verbose': True, 'worker_log': '/home/teuthworker/mnt/teuthology/worker_logs/dispatcher.trial.3318653'} 2026-02-22T21:36:22.297 DEBUG:teuthology.orchestra.run.trial004:> install -d -m0755 -- /home/ubuntu/cephtest/cluster2.data 2026-02-22T21:36:22.300 DEBUG:teuthology.orchestra.run.trial026:> install -d -m0755 -- /home/ubuntu/cephtest/cluster2.data 2026-02-22T21:36:22.318 DEBUG:teuthology.orchestra.run.trial004:> sudo install -d -m0777 -- /var/run/ceph 2026-02-22T21:36:22.355 DEBUG:teuthology.orchestra.run.trial026:> sudo install -d -m0777 -- /var/run/ceph 2026-02-22T21:36:22.382 DEBUG:teuthology.orchestra.run.trial026:> set -ex 2026-02-22T21:36:22.382 DEBUG:teuthology.orchestra.run.trial026:> dd if=/scratch_devs of=/dev/stdout 2026-02-22T21:36:22.439 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2026-02-22T21:36:22.439 DEBUG:teuthology.orchestra.run.trial026:> stat /dev/vg_nvme/lv_1 2026-02-22T21:36:22.494 INFO:teuthology.orchestra.run.trial026.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2026-02-22T21:36:22.494 INFO:teuthology.orchestra.run.trial026.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-02-22T21:36:22.494 INFO:teuthology.orchestra.run.trial026.stdout:Device: 6h/6d Inode: 1298 Links: 1 2026-02-22T21:36:22.494 INFO:teuthology.orchestra.run.trial026.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-02-22T21:36:22.494 INFO:teuthology.orchestra.run.trial026.stdout:Context: system_u:object_r:device_t:s0 2026-02-22T21:36:22.494 INFO:teuthology.orchestra.run.trial026.stdout:Access: 2026-02-22 21:35:43.146116733 +0000 2026-02-22T21:36:22.494 INFO:teuthology.orchestra.run.trial026.stdout:Modify: 2026-02-22 21:35:05.643890351 +0000 2026-02-22T21:36:22.494 INFO:teuthology.orchestra.run.trial026.stdout:Change: 2026-02-22 21:35:05.643890351 +0000 2026-02-22T21:36:22.494 INFO:teuthology.orchestra.run.trial026.stdout: Birth: 2026-02-22 21:35:05.643890351 +0000 2026-02-22T21:36:22.494 DEBUG:teuthology.orchestra.run.trial026:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2026-02-22T21:36:22.556 INFO:teuthology.orchestra.run.trial026.stderr:1+0 records in 2026-02-22T21:36:22.556 INFO:teuthology.orchestra.run.trial026.stderr:1+0 records out 2026-02-22T21:36:22.556 INFO:teuthology.orchestra.run.trial026.stderr:512 bytes copied, 0.000209386 s, 2.4 MB/s 2026-02-22T21:36:22.558 DEBUG:teuthology.orchestra.run.trial026:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2026-02-22T21:36:22.616 DEBUG:teuthology.orchestra.run.trial026:> stat /dev/vg_nvme/lv_2 2026-02-22T21:36:22.672 INFO:teuthology.orchestra.run.trial026.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2026-02-22T21:36:22.672 INFO:teuthology.orchestra.run.trial026.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-02-22T21:36:22.672 INFO:teuthology.orchestra.run.trial026.stdout:Device: 6h/6d Inode: 1291 Links: 1 2026-02-22T21:36:22.672 INFO:teuthology.orchestra.run.trial026.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-02-22T21:36:22.672 INFO:teuthology.orchestra.run.trial026.stdout:Context: system_u:object_r:device_t:s0 2026-02-22T21:36:22.672 INFO:teuthology.orchestra.run.trial026.stdout:Access: 2026-02-22 21:35:43.146116733 +0000 2026-02-22T21:36:22.672 INFO:teuthology.orchestra.run.trial026.stdout:Modify: 2026-02-22 21:35:05.642890389 +0000 2026-02-22T21:36:22.672 INFO:teuthology.orchestra.run.trial026.stdout:Change: 2026-02-22 21:35:05.642890389 +0000 2026-02-22T21:36:22.672 INFO:teuthology.orchestra.run.trial026.stdout: Birth: 2026-02-22 21:35:05.642890389 +0000 2026-02-22T21:36:22.672 DEBUG:teuthology.orchestra.run.trial026:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2026-02-22T21:36:22.733 INFO:teuthology.orchestra.run.trial026.stderr:1+0 records in 2026-02-22T21:36:22.733 INFO:teuthology.orchestra.run.trial026.stderr:1+0 records out 2026-02-22T21:36:22.733 INFO:teuthology.orchestra.run.trial026.stderr:512 bytes copied, 0.000144386 s, 3.5 MB/s 2026-02-22T21:36:22.734 DEBUG:teuthology.orchestra.run.trial026:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2026-02-22T21:36:22.792 DEBUG:teuthology.orchestra.run.trial026:> stat /dev/vg_nvme/lv_3 2026-02-22T21:36:22.849 INFO:teuthology.orchestra.run.trial026.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2026-02-22T21:36:22.849 INFO:teuthology.orchestra.run.trial026.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-02-22T21:36:22.849 INFO:teuthology.orchestra.run.trial026.stdout:Device: 6h/6d Inode: 1300 Links: 1 2026-02-22T21:36:22.849 INFO:teuthology.orchestra.run.trial026.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-02-22T21:36:22.849 INFO:teuthology.orchestra.run.trial026.stdout:Context: system_u:object_r:device_t:s0 2026-02-22T21:36:22.849 INFO:teuthology.orchestra.run.trial026.stdout:Access: 2026-02-22 21:35:43.146116733 +0000 2026-02-22T21:36:22.850 INFO:teuthology.orchestra.run.trial026.stdout:Modify: 2026-02-22 21:35:05.643890351 +0000 2026-02-22T21:36:22.850 INFO:teuthology.orchestra.run.trial026.stdout:Change: 2026-02-22 21:35:05.643890351 +0000 2026-02-22T21:36:22.850 INFO:teuthology.orchestra.run.trial026.stdout: Birth: 2026-02-22 21:35:05.643890351 +0000 2026-02-22T21:36:22.850 DEBUG:teuthology.orchestra.run.trial026:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2026-02-22T21:36:22.912 INFO:teuthology.orchestra.run.trial026.stderr:1+0 records in 2026-02-22T21:36:22.912 INFO:teuthology.orchestra.run.trial026.stderr:1+0 records out 2026-02-22T21:36:22.912 INFO:teuthology.orchestra.run.trial026.stderr:512 bytes copied, 0.000142373 s, 3.6 MB/s 2026-02-22T21:36:22.914 DEBUG:teuthology.orchestra.run.trial026:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2026-02-22T21:36:22.971 DEBUG:teuthology.orchestra.run.trial026:> stat /dev/vg_nvme/lv_4 2026-02-22T21:36:23.027 INFO:teuthology.orchestra.run.trial026.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2026-02-22T21:36:23.027 INFO:teuthology.orchestra.run.trial026.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-02-22T21:36:23.027 INFO:teuthology.orchestra.run.trial026.stdout:Device: 6h/6d Inode: 1294 Links: 1 2026-02-22T21:36:23.027 INFO:teuthology.orchestra.run.trial026.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-02-22T21:36:23.027 INFO:teuthology.orchestra.run.trial026.stdout:Context: system_u:object_r:device_t:s0 2026-02-22T21:36:23.027 INFO:teuthology.orchestra.run.trial026.stdout:Access: 2026-02-22 21:35:43.146116733 +0000 2026-02-22T21:36:23.027 INFO:teuthology.orchestra.run.trial026.stdout:Modify: 2026-02-22 21:35:05.643890351 +0000 2026-02-22T21:36:23.027 INFO:teuthology.orchestra.run.trial026.stdout:Change: 2026-02-22 21:35:05.643890351 +0000 2026-02-22T21:36:23.028 INFO:teuthology.orchestra.run.trial026.stdout: Birth: 2026-02-22 21:35:05.643890351 +0000 2026-02-22T21:36:23.028 DEBUG:teuthology.orchestra.run.trial026:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2026-02-22T21:36:23.090 INFO:teuthology.orchestra.run.trial026.stderr:1+0 records in 2026-02-22T21:36:23.090 INFO:teuthology.orchestra.run.trial026.stderr:1+0 records out 2026-02-22T21:36:23.090 INFO:teuthology.orchestra.run.trial026.stderr:512 bytes copied, 0.000135009 s, 3.8 MB/s 2026-02-22T21:36:23.091 DEBUG:teuthology.orchestra.run.trial026:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2026-02-22T21:36:23.147 INFO:tasks.ceph:osd dev map: {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'} 2026-02-22T21:36:23.148 INFO:tasks.ceph:remote_to_roles_to_devs: {Remote(name='ubuntu@trial026.front.sepia.ceph.com'): {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'}} 2026-02-22T21:36:23.148 INFO:tasks.ceph:Generating config... 2026-02-22T21:36:23.150 INFO:tasks.ceph:[client] debug rbd = 20 2026-02-22T21:36:23.150 INFO:tasks.ceph:[client] debug rbd_mirror = 15 2026-02-22T21:36:23.150 INFO:tasks.ceph:[client] log to stderr = False 2026-02-22T21:36:23.150 INFO:tasks.ceph:[client] rbd default features = 125 2026-02-22T21:36:23.150 INFO:tasks.ceph:[client] rbd mirror image policy type = simple 2026-02-22T21:36:23.150 INFO:tasks.ceph:[client.mirror.0] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T21:36:23.150 INFO:tasks.ceph:[client.mirror.0] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T21:36:23.151 INFO:tasks.ceph:[client.mirror.1] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T21:36:23.151 INFO:tasks.ceph:[client.mirror.1] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T21:36:23.151 INFO:tasks.ceph:[client.mirror.2] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T21:36:23.151 INFO:tasks.ceph:[client.mirror.2] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T21:36:23.151 INFO:tasks.ceph:[client.mirror.3] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T21:36:23.151 INFO:tasks.ceph:[client.mirror.3] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T21:36:23.151 INFO:tasks.ceph:[global] mon client directed command retry = 5 2026-02-22T21:36:23.151 INFO:tasks.ceph:[global] mon warn on pool no app = False 2026-02-22T21:36:23.151 INFO:tasks.ceph:[global] ms inject socket failures = 5000 2026-02-22T21:36:23.151 INFO:tasks.ceph:[mgr] debug mgr = 20 2026-02-22T21:36:23.151 INFO:tasks.ceph:[mgr] debug ms = 1 2026-02-22T21:36:23.152 INFO:tasks.ceph:[mon] debug mon = 20 2026-02-22T21:36:23.152 INFO:tasks.ceph:[mon] debug ms = 1 2026-02-22T21:36:23.152 INFO:tasks.ceph:[mon] debug paxos = 20 2026-02-22T21:36:23.152 INFO:tasks.ceph:[osd] bluestore block size = 96636764160 2026-02-22T21:36:23.152 INFO:tasks.ceph:[osd] bluestore compression algorithm = lz4 2026-02-22T21:36:23.152 INFO:tasks.ceph:[osd] bluestore compression mode = aggressive 2026-02-22T21:36:23.152 INFO:tasks.ceph:[osd] bluestore fsck on mount = True 2026-02-22T21:36:23.152 INFO:tasks.ceph:[osd] debug bluefs = 1/20 2026-02-22T21:36:23.152 INFO:tasks.ceph:[osd] debug bluestore = 1/20 2026-02-22T21:36:23.152 INFO:tasks.ceph:[osd] debug ms = 1 2026-02-22T21:36:23.152 INFO:tasks.ceph:[osd] debug osd = 20 2026-02-22T21:36:23.152 INFO:tasks.ceph:[osd] debug rocksdb = 4/10 2026-02-22T21:36:23.153 INFO:tasks.ceph:[osd] mon osd backfillfull_ratio = 0.85 2026-02-22T21:36:23.153 INFO:tasks.ceph:[osd] mon osd full ratio = 0.9 2026-02-22T21:36:23.153 INFO:tasks.ceph:[osd] mon osd nearfull ratio = 0.8 2026-02-22T21:36:23.153 INFO:tasks.ceph:[osd] osd failsafe full ratio = 0.95 2026-02-22T21:36:23.153 INFO:tasks.ceph:[osd] osd objectstore = bluestore 2026-02-22T21:36:23.153 INFO:tasks.ceph:Setting up cluster2.mon.a... 2026-02-22T21:36:23.153 DEBUG:teuthology.orchestra.run.trial026:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring /etc/ceph/cluster2.keyring 2026-02-22T21:36:23.223 INFO:teuthology.orchestra.run.trial026.stdout:creating /etc/ceph/cluster2.keyring 2026-02-22T21:36:23.226 DEBUG:teuthology.orchestra.run.trial026:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=mon. /etc/ceph/cluster2.keyring 2026-02-22T21:36:23.301 DEBUG:teuthology.orchestra.run.trial026:> sudo chmod 0644 /etc/ceph/cluster2.keyring 2026-02-22T21:36:23.366 DEBUG:tasks.ceph:Ceph mon addresses: [('cluster2.mon.a', '10.20.193.26')] 2026-02-22T21:36:23.366 DEBUG:tasks.ceph:writing out conf {'global': {'chdir': '', 'pid file': '/var/run/ceph/$cluster-$name.pid', 'auth supported': 'cephx', 'filestore xattr use omap': 'true', 'mon clock drift allowed': '1.000', 'osd crush chooseleaf type': '0', 'auth debug': 'true', 'ms die on old message': 'true', 'ms die on bug': 'true', 'mon max pg per osd': '10000', 'mon pg warn max object skew': '0', 'osd_pool_default_pg_autoscale_mode': 'off', 'osd pool default size': '2', 'mon osd allow primary affinity': 'true', 'mon osd allow pg remap': 'true', 'mon warn on legacy crush tunables': 'false', 'mon warn on crush straw calc version zero': 'false', 'mon warn on no sortbitwise': 'false', 'mon warn on osd down out interval zero': 'false', 'mon warn on too few osds': 'false', 'mon_warn_on_pool_pg_num_not_power_of_two': 'false', 'mon_warn_on_pool_no_redundancy': 'false', 'mon_allow_pool_size_one': 'true', 'osd pool default erasure code profile': 'plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd', 'osd default data pool replay window': '5', 'mon allow pool delete': 'true', 'mon cluster log file level': 'debug', 'debug asserts on shutdown': 'true', 'mon health detail to clog': 'false', 'mon host': '10.20.193.26', 'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'osd': {'osd journal size': '100', 'osd scrub load threshold': '5.0', 'osd scrub max interval': '600', 'osd mclock profile': 'high_recovery_ops', 'osd mclock skip benchmark': 'true', 'osd recover clone overlap': 'true', 'osd recovery max chunk': '1048576', 'osd debug shutdown': 'true', 'osd debug op order': 'true', 'osd debug verify stray on activate': 'true', 'osd debug trim objects': 'true', 'osd open classes on start': 'true', 'osd debug pg log writeout': 'true', 'osd deep scrub update digest min age': '30', 'osd map max advance': '10', 'journal zero on create': 'true', 'filestore ondisk finisher threads': '3', 'filestore apply finisher threads': '3', 'bdev debug aio': 'true', 'osd debug misdirected ops': 'true', 'bluestore block size': 96636764160, 'bluestore compression algorithm': 'lz4', 'bluestore compression mode': 'aggressive', 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}, 'mgr': {'debug ms': 1, 'debug mgr': 20, 'debug mon': '20', 'debug auth': '20', 'mon reweight min pgs per osd': '4', 'mon reweight min bytes per osd': '10', 'mgr/telemetry/nag': 'false'}, 'mon': {'debug ms': 1, 'debug mon': 20, 'debug paxos': 20, 'debug auth': '20', 'mon data avail warn': '5', 'mon mgr mkfs grace': '240', 'mon reweight min pgs per osd': '4', 'mon osd reporter subtree level': 'osd', 'mon osd prime pg temp': 'true', 'mon reweight min bytes per osd': '10', 'auth mon ticket ttl': '660', 'auth service ticket ttl': '240', 'mon_warn_on_insecure_global_id_reclaim': 'false', 'mon_warn_on_insecure_global_id_reclaim_allowed': 'false', 'mon_down_mkfs_grace': '2m', 'mon_warn_on_filestore_osds': 'false'}, 'client': {'rgw cache enabled': 'true', 'rgw enable ops log': 'true', 'rgw enable usage log': 'true', 'log file': '/var/log/ceph/$cluster-$name.$pid.log', 'admin socket': '/var/run/ceph/$cluster-$name.$pid.asok', 'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'simple'}, 'mon.a': {}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}} 2026-02-22T21:36:23.368 DEBUG:teuthology.orchestra.run.trial026:> set -ex 2026-02-22T21:36:23.368 DEBUG:teuthology.orchestra.run.trial026:> dd of=/home/ubuntu/cephtest/ceph.tmp.conf 2026-02-22T21:36:23.424 DEBUG:teuthology.orchestra.run.trial026:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage monmaptool -c /home/ubuntu/cephtest/ceph.tmp.conf --create --clobber --enable-all-features --add a 10.20.193.26 --print /home/ubuntu/cephtest/cluster2.monmap 2026-02-22T21:36:23.493 INFO:teuthology.orchestra.run.trial026.stdout:monmaptool: monmap file /home/ubuntu/cephtest/cluster2.monmap 2026-02-22T21:36:23.494 INFO:teuthology.orchestra.run.trial026.stdout:monmaptool: generated fsid 2e52b784-3eb2-4856-a6f6-868b97c4e845 2026-02-22T21:36:23.494 INFO:teuthology.orchestra.run.trial026.stdout:setting min_mon_release = quincy 2026-02-22T21:36:23.494 INFO:teuthology.orchestra.run.trial026.stdout:epoch 0 2026-02-22T21:36:23.494 INFO:teuthology.orchestra.run.trial026.stdout:fsid 2e52b784-3eb2-4856-a6f6-868b97c4e845 2026-02-22T21:36:23.494 INFO:teuthology.orchestra.run.trial026.stdout:last_changed 2026-02-22T21:36:23.492507+0000 2026-02-22T21:36:23.494 INFO:teuthology.orchestra.run.trial026.stdout:created 2026-02-22T21:36:23.492507+0000 2026-02-22T21:36:23.494 INFO:teuthology.orchestra.run.trial026.stdout:min_mon_release 17 (quincy) 2026-02-22T21:36:23.494 INFO:teuthology.orchestra.run.trial026.stdout:election_strategy: 1 2026-02-22T21:36:23.494 INFO:teuthology.orchestra.run.trial026.stdout:0: [v2:10.20.193.26:3300/0,v1:10.20.193.26:6789/0] mon.a 2026-02-22T21:36:23.494 INFO:teuthology.orchestra.run.trial026.stdout:monmaptool: writing epoch 0 to /home/ubuntu/cephtest/cluster2.monmap (1 monitors) 2026-02-22T21:36:23.496 DEBUG:teuthology.orchestra.run.trial026:> rm -- /home/ubuntu/cephtest/ceph.tmp.conf 2026-02-22T21:36:23.553 INFO:tasks.ceph:Writing /etc/ceph/cluster2.conf for FSID 2e52b784-3eb2-4856-a6f6-868b97c4e845... 2026-02-22T21:36:23.555 DEBUG:teuthology.orchestra.run.trial004:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/cluster2.conf && sudo chmod 0644 /etc/ceph/cluster2.conf > /dev/null 2026-02-22T21:36:23.557 DEBUG:teuthology.orchestra.run.trial026:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/cluster2.conf && sudo chmod 0644 /etc/ceph/cluster2.conf > /dev/null 2026-02-22T21:36:23.597 INFO:teuthology.orchestra.run.trial004.stdout:[global] 2026-02-22T21:36:23.597 INFO:teuthology.orchestra.run.trial004.stdout: chdir = "" 2026-02-22T21:36:23.597 INFO:teuthology.orchestra.run.trial004.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2026-02-22T21:36:23.598 INFO:teuthology.orchestra.run.trial004.stdout: auth supported = cephx 2026-02-22T21:36:23.598 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:36:23.598 INFO:teuthology.orchestra.run.trial004.stdout: filestore xattr use omap = true 2026-02-22T21:36:23.598 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:36:23.598 INFO:teuthology.orchestra.run.trial004.stdout: mon clock drift allowed = 1.000 2026-02-22T21:36:23.598 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:36:23.598 INFO:teuthology.orchestra.run.trial004.stdout: osd crush chooseleaf type = 0 2026-02-22T21:36:23.598 INFO:teuthology.orchestra.run.trial004.stdout: auth debug = true 2026-02-22T21:36:23.598 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:36:23.598 INFO:teuthology.orchestra.run.trial004.stdout: ms die on old message = true 2026-02-22T21:36:23.599 INFO:teuthology.orchestra.run.trial004.stdout: ms die on bug = true 2026-02-22T21:36:23.599 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:36:23.599 INFO:teuthology.orchestra.run.trial004.stdout: mon max pg per osd = 10000 # >= luminous 2026-02-22T21:36:23.599 INFO:teuthology.orchestra.run.trial004.stdout: mon pg warn max object skew = 0 2026-02-22T21:36:23.599 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:36:23.599 INFO:teuthology.orchestra.run.trial004.stdout: # disable pg_autoscaler by default for new pools 2026-02-22T21:36:23.599 INFO:teuthology.orchestra.run.trial004.stdout: osd_pool_default_pg_autoscale_mode = off 2026-02-22T21:36:23.599 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:36:23.599 INFO:teuthology.orchestra.run.trial004.stdout: osd pool default size = 2 2026-02-22T21:36:23.599 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:36:23.599 INFO:teuthology.orchestra.run.trial004.stdout: mon osd allow primary affinity = true 2026-02-22T21:36:23.600 INFO:teuthology.orchestra.run.trial004.stdout: mon osd allow pg remap = true 2026-02-22T21:36:23.600 INFO:teuthology.orchestra.run.trial004.stdout: mon warn on legacy crush tunables = false 2026-02-22T21:36:23.600 INFO:teuthology.orchestra.run.trial004.stdout: mon warn on crush straw calc version zero = false 2026-02-22T21:36:23.600 INFO:teuthology.orchestra.run.trial004.stdout: mon warn on no sortbitwise = false 2026-02-22T21:36:23.600 INFO:teuthology.orchestra.run.trial004.stdout: mon warn on osd down out interval zero = false 2026-02-22T21:36:23.600 INFO:teuthology.orchestra.run.trial004.stdout: mon warn on too few osds = false 2026-02-22T21:36:23.600 INFO:teuthology.orchestra.run.trial004.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2026-02-22T21:36:23.600 INFO:teuthology.orchestra.run.trial004.stdout: mon_warn_on_pool_no_redundancy = false 2026-02-22T21:36:23.600 INFO:teuthology.orchestra.run.trial004.stdout: mon_allow_pool_size_one = true 2026-02-22T21:36:23.600 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:36:23.600 INFO:teuthology.orchestra.run.trial004.stdout: osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2026-02-22T21:36:23.601 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:36:23.601 INFO:teuthology.orchestra.run.trial004.stdout: osd default data pool replay window = 5 2026-02-22T21:36:23.601 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:36:23.601 INFO:teuthology.orchestra.run.trial004.stdout: mon allow pool delete = true 2026-02-22T21:36:23.601 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:36:23.601 INFO:teuthology.orchestra.run.trial004.stdout: mon cluster log file level = debug 2026-02-22T21:36:23.601 INFO:teuthology.orchestra.run.trial004.stdout: debug asserts on shutdown = true 2026-02-22T21:36:23.601 INFO:teuthology.orchestra.run.trial004.stdout: mon health detail to clog = false 2026-02-22T21:36:23.601 INFO:teuthology.orchestra.run.trial004.stdout: mon host = 10.20.193.26 2026-02-22T21:36:23.601 INFO:teuthology.orchestra.run.trial004.stdout: mon client directed command retry = 5 2026-02-22T21:36:23.601 INFO:teuthology.orchestra.run.trial004.stdout: mon warn on pool no app = False 2026-02-22T21:36:23.601 INFO:teuthology.orchestra.run.trial004.stdout: ms inject socket failures = 5000 2026-02-22T21:36:23.602 INFO:teuthology.orchestra.run.trial004.stdout: fsid = 2e52b784-3eb2-4856-a6f6-868b97c4e845 2026-02-22T21:36:23.602 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:36:23.602 INFO:teuthology.orchestra.run.trial004.stdout:[osd] 2026-02-22T21:36:23.602 INFO:teuthology.orchestra.run.trial004.stdout: osd journal size = 100 2026-02-22T21:36:23.602 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:36:23.602 INFO:teuthology.orchestra.run.trial004.stdout: osd scrub load threshold = 5.0 2026-02-22T21:36:23.602 INFO:teuthology.orchestra.run.trial004.stdout: osd scrub max interval = 600 2026-02-22T21:36:23.602 INFO:teuthology.orchestra.run.trial004.stdout: osd mclock profile = high_recovery_ops 2026-02-22T21:36:23.602 INFO:teuthology.orchestra.run.trial004.stdout: osd mclock skip benchmark = true 2026-02-22T21:36:23.602 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:36:23.603 INFO:teuthology.orchestra.run.trial004.stdout: osd recover clone overlap = true 2026-02-22T21:36:23.603 INFO:teuthology.orchestra.run.trial004.stdout: osd recovery max chunk = 1048576 2026-02-22T21:36:23.603 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:36:23.603 INFO:teuthology.orchestra.run.trial004.stdout: osd debug shutdown = true 2026-02-22T21:36:23.603 INFO:teuthology.orchestra.run.trial004.stdout: osd debug op order = true 2026-02-22T21:36:23.603 INFO:teuthology.orchestra.run.trial004.stdout: osd debug verify stray on activate = true 2026-02-22T21:36:23.603 INFO:teuthology.orchestra.run.trial004.stdout: osd debug trim objects = true 2026-02-22T21:36:23.603 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:36:23.603 INFO:teuthology.orchestra.run.trial004.stdout: osd open classes on start = true 2026-02-22T21:36:23.603 INFO:teuthology.orchestra.run.trial004.stdout: osd debug pg log writeout = true 2026-02-22T21:36:23.603 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:36:23.603 INFO:teuthology.orchestra.run.trial004.stdout: osd deep scrub update digest min age = 30 2026-02-22T21:36:23.604 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:36:23.604 INFO:teuthology.orchestra.run.trial004.stdout: osd map max advance = 10 2026-02-22T21:36:23.604 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:36:23.604 INFO:teuthology.orchestra.run.trial004.stdout: journal zero on create = true 2026-02-22T21:36:23.604 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:36:23.604 INFO:teuthology.orchestra.run.trial004.stdout: filestore ondisk finisher threads = 3 2026-02-22T21:36:23.604 INFO:teuthology.orchestra.run.trial004.stdout: filestore apply finisher threads = 3 2026-02-22T21:36:23.604 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:36:23.604 INFO:teuthology.orchestra.run.trial004.stdout: bdev debug aio = true 2026-02-22T21:36:23.604 INFO:teuthology.orchestra.run.trial004.stdout: osd debug misdirected ops = true 2026-02-22T21:36:23.604 INFO:teuthology.orchestra.run.trial004.stdout: bluestore block size = 96636764160 2026-02-22T21:36:23.604 INFO:teuthology.orchestra.run.trial004.stdout: bluestore compression algorithm = lz4 2026-02-22T21:36:23.604 INFO:teuthology.orchestra.run.trial004.stdout: bluestore compression mode = aggressive 2026-02-22T21:36:23.605 INFO:teuthology.orchestra.run.trial004.stdout: bluestore fsck on mount = True 2026-02-22T21:36:23.605 INFO:teuthology.orchestra.run.trial004.stdout: debug bluefs = 1/20 2026-02-22T21:36:23.605 INFO:teuthology.orchestra.run.trial004.stdout: debug bluestore = 1/20 2026-02-22T21:36:23.605 INFO:teuthology.orchestra.run.trial004.stdout: debug ms = 1 2026-02-22T21:36:23.605 INFO:teuthology.orchestra.run.trial004.stdout: debug osd = 20 2026-02-22T21:36:23.605 INFO:teuthology.orchestra.run.trial004.stdout: debug rocksdb = 4/10 2026-02-22T21:36:23.605 INFO:teuthology.orchestra.run.trial004.stdout: mon osd backfillfull_ratio = 0.85 2026-02-22T21:36:23.605 INFO:teuthology.orchestra.run.trial004.stdout: mon osd full ratio = 0.9 2026-02-22T21:36:23.605 INFO:teuthology.orchestra.run.trial004.stdout: mon osd nearfull ratio = 0.8 2026-02-22T21:36:23.605 INFO:teuthology.orchestra.run.trial004.stdout: osd failsafe full ratio = 0.95 2026-02-22T21:36:23.605 INFO:teuthology.orchestra.run.trial004.stdout: osd objectstore = bluestore 2026-02-22T21:36:23.605 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:36:23.606 INFO:teuthology.orchestra.run.trial004.stdout:[mgr] 2026-02-22T21:36:23.606 INFO:teuthology.orchestra.run.trial004.stdout: debug ms = 1 2026-02-22T21:36:23.606 INFO:teuthology.orchestra.run.trial004.stdout: debug mgr = 20 2026-02-22T21:36:23.606 INFO:teuthology.orchestra.run.trial004.stdout: debug mon = 20 2026-02-22T21:36:23.606 INFO:teuthology.orchestra.run.trial004.stdout: debug auth = 20 2026-02-22T21:36:23.606 INFO:teuthology.orchestra.run.trial004.stdout: mon reweight min pgs per osd = 4 2026-02-22T21:36:23.606 INFO:teuthology.orchestra.run.trial004.stdout: mon reweight min bytes per osd = 10 2026-02-22T21:36:23.606 INFO:teuthology.orchestra.run.trial004.stdout: mgr/telemetry/nag = false 2026-02-22T21:36:23.606 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:36:23.606 INFO:teuthology.orchestra.run.trial004.stdout:[mon] 2026-02-22T21:36:23.606 INFO:teuthology.orchestra.run.trial004.stdout: debug ms = 1 2026-02-22T21:36:23.606 INFO:teuthology.orchestra.run.trial004.stdout: debug mon = 20 2026-02-22T21:36:23.606 INFO:teuthology.orchestra.run.trial004.stdout: debug paxos = 20 2026-02-22T21:36:23.607 INFO:teuthology.orchestra.run.trial004.stdout: debug auth = 20 2026-02-22T21:36:23.607 INFO:teuthology.orchestra.run.trial004.stdout: mon data avail warn = 5 2026-02-22T21:36:23.607 INFO:teuthology.orchestra.run.trial004.stdout: mon mgr mkfs grace = 240 2026-02-22T21:36:23.607 INFO:teuthology.orchestra.run.trial004.stdout: mon reweight min pgs per osd = 4 2026-02-22T21:36:23.607 INFO:teuthology.orchestra.run.trial004.stdout: mon osd reporter subtree level = osd 2026-02-22T21:36:23.607 INFO:teuthology.orchestra.run.trial004.stdout: mon osd prime pg temp = true 2026-02-22T21:36:23.607 INFO:teuthology.orchestra.run.trial004.stdout: mon reweight min bytes per osd = 10 2026-02-22T21:36:23.607 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:36:23.607 INFO:teuthology.orchestra.run.trial004.stdout: # rotate auth tickets quickly to exercise renewal paths 2026-02-22T21:36:23.607 INFO:teuthology.orchestra.run.trial004.stdout: auth mon ticket ttl = 660 # 11m 2026-02-22T21:36:23.607 INFO:teuthology.orchestra.run.trial004.stdout: auth service ticket ttl = 240 # 4m 2026-02-22T21:36:23.607 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:36:23.608 INFO:teuthology.orchestra.run.trial004.stdout: # don't complain about insecure global_id in the test suite 2026-02-22T21:36:23.608 INFO:teuthology.orchestra.run.trial004.stdout: mon_warn_on_insecure_global_id_reclaim = false 2026-02-22T21:36:23.608 INFO:teuthology.orchestra.run.trial004.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2026-02-22T21:36:23.608 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:36:23.608 INFO:teuthology.orchestra.run.trial004.stdout: # 1m isn't quite enough 2026-02-22T21:36:23.608 INFO:teuthology.orchestra.run.trial004.stdout: mon_down_mkfs_grace = 2m 2026-02-22T21:36:23.608 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:36:23.608 INFO:teuthology.orchestra.run.trial004.stdout: mon_warn_on_filestore_osds = false 2026-02-22T21:36:23.608 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T21:36:23.608 INFO:teuthology.orchestra.run.trial004.stdout:[client] 2026-02-22T21:36:23.608 INFO:teuthology.orchestra.run.trial004.stdout: rgw cache enabled = true 2026-02-22T21:36:23.608 INFO:teuthology.orchestra.run.trial004.stdout: rgw enable ops log = true 2026-02-22T21:36:23.608 INFO:teuthology.orchestra.run.trial004.stdout: rgw enable usage log = true 2026-02-22T21:36:23.609 INFO:teuthology.orchestra.run.trial004.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2026-02-22T21:36:23.609 INFO:teuthology.orchestra.run.trial004.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2026-02-22T21:36:23.609 INFO:teuthology.orchestra.run.trial004.stdout: debug rbd = 20 2026-02-22T21:36:23.609 INFO:teuthology.orchestra.run.trial004.stdout: debug rbd_mirror = 15 2026-02-22T21:36:23.609 INFO:teuthology.orchestra.run.trial004.stdout: log to stderr = False 2026-02-22T21:36:23.609 INFO:teuthology.orchestra.run.trial004.stdout: rbd default features = 125 2026-02-22T21:36:23.609 INFO:teuthology.orchestra.run.trial004.stdout: rbd mirror image policy type = simple 2026-02-22T21:36:23.609 INFO:teuthology.orchestra.run.trial004.stdout:[mon.a] 2026-02-22T21:36:23.609 INFO:teuthology.orchestra.run.trial004.stdout:[client.mirror.0] 2026-02-22T21:36:23.609 INFO:teuthology.orchestra.run.trial004.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T21:36:23.609 INFO:teuthology.orchestra.run.trial004.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T21:36:23.609 INFO:teuthology.orchestra.run.trial004.stdout:[client.mirror.1] 2026-02-22T21:36:23.609 INFO:teuthology.orchestra.run.trial004.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T21:36:23.610 INFO:teuthology.orchestra.run.trial004.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T21:36:23.610 INFO:teuthology.orchestra.run.trial004.stdout:[client.mirror.2] 2026-02-22T21:36:23.610 INFO:teuthology.orchestra.run.trial004.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T21:36:23.610 INFO:teuthology.orchestra.run.trial004.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T21:36:23.610 INFO:teuthology.orchestra.run.trial004.stdout:[client.mirror.3] 2026-02-22T21:36:23.610 INFO:teuthology.orchestra.run.trial004.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T21:36:23.610 INFO:teuthology.orchestra.run.trial004.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T21:36:23.635 INFO:teuthology.orchestra.run.trial026.stdout:[global] 2026-02-22T21:36:23.635 INFO:teuthology.orchestra.run.trial026.stdout: chdir = "" 2026-02-22T21:36:23.635 INFO:teuthology.orchestra.run.trial026.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2026-02-22T21:36:23.635 INFO:teuthology.orchestra.run.trial026.stdout: auth supported = cephx 2026-02-22T21:36:23.635 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:36:23.635 INFO:teuthology.orchestra.run.trial026.stdout: filestore xattr use omap = true 2026-02-22T21:36:23.635 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:36:23.635 INFO:teuthology.orchestra.run.trial026.stdout: mon clock drift allowed = 1.000 2026-02-22T21:36:23.636 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:36:23.636 INFO:teuthology.orchestra.run.trial026.stdout: osd crush chooseleaf type = 0 2026-02-22T21:36:23.636 INFO:teuthology.orchestra.run.trial026.stdout: auth debug = true 2026-02-22T21:36:23.636 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:36:23.636 INFO:teuthology.orchestra.run.trial026.stdout: ms die on old message = true 2026-02-22T21:36:23.636 INFO:teuthology.orchestra.run.trial026.stdout: ms die on bug = true 2026-02-22T21:36:23.636 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:36:23.636 INFO:teuthology.orchestra.run.trial026.stdout: mon max pg per osd = 10000 # >= luminous 2026-02-22T21:36:23.636 INFO:teuthology.orchestra.run.trial026.stdout: mon pg warn max object skew = 0 2026-02-22T21:36:23.636 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:36:23.636 INFO:teuthology.orchestra.run.trial026.stdout: # disable pg_autoscaler by default for new pools 2026-02-22T21:36:23.637 INFO:teuthology.orchestra.run.trial026.stdout: osd_pool_default_pg_autoscale_mode = off 2026-02-22T21:36:23.637 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:36:23.637 INFO:teuthology.orchestra.run.trial026.stdout: osd pool default size = 2 2026-02-22T21:36:23.637 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:36:23.637 INFO:teuthology.orchestra.run.trial026.stdout: mon osd allow primary affinity = true 2026-02-22T21:36:23.637 INFO:teuthology.orchestra.run.trial026.stdout: mon osd allow pg remap = true 2026-02-22T21:36:23.637 INFO:teuthology.orchestra.run.trial026.stdout: mon warn on legacy crush tunables = false 2026-02-22T21:36:23.637 INFO:teuthology.orchestra.run.trial026.stdout: mon warn on crush straw calc version zero = false 2026-02-22T21:36:23.637 INFO:teuthology.orchestra.run.trial026.stdout: mon warn on no sortbitwise = false 2026-02-22T21:36:23.637 INFO:teuthology.orchestra.run.trial026.stdout: mon warn on osd down out interval zero = false 2026-02-22T21:36:23.637 INFO:teuthology.orchestra.run.trial026.stdout: mon warn on too few osds = false 2026-02-22T21:36:23.637 INFO:teuthology.orchestra.run.trial026.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2026-02-22T21:36:23.638 INFO:teuthology.orchestra.run.trial026.stdout: mon_warn_on_pool_no_redundancy = false 2026-02-22T21:36:23.638 INFO:teuthology.orchestra.run.trial026.stdout: mon_allow_pool_size_one = true 2026-02-22T21:36:23.638 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:36:23.638 INFO:teuthology.orchestra.run.trial026.stdout: osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2026-02-22T21:36:23.638 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:36:23.638 INFO:teuthology.orchestra.run.trial026.stdout: osd default data pool replay window = 5 2026-02-22T21:36:23.638 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:36:23.638 INFO:teuthology.orchestra.run.trial026.stdout: mon allow pool delete = true 2026-02-22T21:36:23.638 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:36:23.638 INFO:teuthology.orchestra.run.trial026.stdout: mon cluster log file level = debug 2026-02-22T21:36:23.638 INFO:teuthology.orchestra.run.trial026.stdout: debug asserts on shutdown = true 2026-02-22T21:36:23.638 INFO:teuthology.orchestra.run.trial026.stdout: mon health detail to clog = false 2026-02-22T21:36:23.638 INFO:teuthology.orchestra.run.trial026.stdout: mon host = 10.20.193.26 2026-02-22T21:36:23.639 INFO:teuthology.orchestra.run.trial026.stdout: mon client directed command retry = 5 2026-02-22T21:36:23.639 INFO:teuthology.orchestra.run.trial026.stdout: mon warn on pool no app = False 2026-02-22T21:36:23.639 INFO:teuthology.orchestra.run.trial026.stdout: ms inject socket failures = 5000 2026-02-22T21:36:23.639 INFO:teuthology.orchestra.run.trial026.stdout: fsid = 2e52b784-3eb2-4856-a6f6-868b97c4e845 2026-02-22T21:36:23.639 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:36:23.639 INFO:teuthology.orchestra.run.trial026.stdout:[osd] 2026-02-22T21:36:23.639 INFO:teuthology.orchestra.run.trial026.stdout: osd journal size = 100 2026-02-22T21:36:23.639 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:36:23.639 INFO:teuthology.orchestra.run.trial026.stdout: osd scrub load threshold = 5.0 2026-02-22T21:36:23.639 INFO:teuthology.orchestra.run.trial026.stdout: osd scrub max interval = 600 2026-02-22T21:36:23.639 INFO:teuthology.orchestra.run.trial026.stdout: osd mclock profile = high_recovery_ops 2026-02-22T21:36:23.639 INFO:teuthology.orchestra.run.trial026.stdout: osd mclock skip benchmark = true 2026-02-22T21:36:23.639 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:36:23.640 INFO:teuthology.orchestra.run.trial026.stdout: osd recover clone overlap = true 2026-02-22T21:36:23.640 INFO:teuthology.orchestra.run.trial026.stdout: osd recovery max chunk = 1048576 2026-02-22T21:36:23.640 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:36:23.640 INFO:teuthology.orchestra.run.trial026.stdout: osd debug shutdown = true 2026-02-22T21:36:23.640 INFO:teuthology.orchestra.run.trial026.stdout: osd debug op order = true 2026-02-22T21:36:23.640 INFO:teuthology.orchestra.run.trial026.stdout: osd debug verify stray on activate = true 2026-02-22T21:36:23.640 INFO:teuthology.orchestra.run.trial026.stdout: osd debug trim objects = true 2026-02-22T21:36:23.640 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:36:23.640 INFO:teuthology.orchestra.run.trial026.stdout: osd open classes on start = true 2026-02-22T21:36:23.640 INFO:teuthology.orchestra.run.trial026.stdout: osd debug pg log writeout = true 2026-02-22T21:36:23.640 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:36:23.640 INFO:teuthology.orchestra.run.trial026.stdout: osd deep scrub update digest min age = 30 2026-02-22T21:36:23.641 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:36:23.641 INFO:teuthology.orchestra.run.trial026.stdout: osd map max advance = 10 2026-02-22T21:36:23.641 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:36:23.641 INFO:teuthology.orchestra.run.trial026.stdout: journal zero on create = true 2026-02-22T21:36:23.641 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:36:23.641 INFO:teuthology.orchestra.run.trial026.stdout: filestore ondisk finisher threads = 3 2026-02-22T21:36:23.641 INFO:teuthology.orchestra.run.trial026.stdout: filestore apply finisher threads = 3 2026-02-22T21:36:23.641 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:36:23.641 INFO:teuthology.orchestra.run.trial026.stdout: bdev debug aio = true 2026-02-22T21:36:23.641 INFO:teuthology.orchestra.run.trial026.stdout: osd debug misdirected ops = true 2026-02-22T21:36:23.641 INFO:teuthology.orchestra.run.trial026.stdout: bluestore block size = 96636764160 2026-02-22T21:36:23.641 INFO:teuthology.orchestra.run.trial026.stdout: bluestore compression algorithm = lz4 2026-02-22T21:36:23.641 INFO:teuthology.orchestra.run.trial026.stdout: bluestore compression mode = aggressive 2026-02-22T21:36:23.642 INFO:teuthology.orchestra.run.trial026.stdout: bluestore fsck on mount = True 2026-02-22T21:36:23.642 INFO:teuthology.orchestra.run.trial026.stdout: debug bluefs = 1/20 2026-02-22T21:36:23.642 INFO:teuthology.orchestra.run.trial026.stdout: debug bluestore = 1/20 2026-02-22T21:36:23.642 INFO:teuthology.orchestra.run.trial026.stdout: debug ms = 1 2026-02-22T21:36:23.642 INFO:teuthology.orchestra.run.trial026.stdout: debug osd = 20 2026-02-22T21:36:23.642 INFO:teuthology.orchestra.run.trial026.stdout: debug rocksdb = 4/10 2026-02-22T21:36:23.642 INFO:teuthology.orchestra.run.trial026.stdout: mon osd backfillfull_ratio = 0.85 2026-02-22T21:36:23.642 INFO:teuthology.orchestra.run.trial026.stdout: mon osd full ratio = 0.9 2026-02-22T21:36:23.642 INFO:teuthology.orchestra.run.trial026.stdout: mon osd nearfull ratio = 0.8 2026-02-22T21:36:23.642 INFO:teuthology.orchestra.run.trial026.stdout: osd failsafe full ratio = 0.95 2026-02-22T21:36:23.642 INFO:teuthology.orchestra.run.trial026.stdout: osd objectstore = bluestore 2026-02-22T21:36:23.642 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:36:23.643 INFO:teuthology.orchestra.run.trial026.stdout:[mgr] 2026-02-22T21:36:23.643 INFO:teuthology.orchestra.run.trial026.stdout: debug ms = 1 2026-02-22T21:36:23.643 INFO:teuthology.orchestra.run.trial026.stdout: debug mgr = 20 2026-02-22T21:36:23.643 INFO:teuthology.orchestra.run.trial026.stdout: debug mon = 20 2026-02-22T21:36:23.643 INFO:teuthology.orchestra.run.trial026.stdout: debug auth = 20 2026-02-22T21:36:23.643 INFO:teuthology.orchestra.run.trial026.stdout: mon reweight min pgs per osd = 4 2026-02-22T21:36:23.643 INFO:teuthology.orchestra.run.trial026.stdout: mon reweight min bytes per osd = 10 2026-02-22T21:36:23.643 INFO:teuthology.orchestra.run.trial026.stdout: mgr/telemetry/nag = false 2026-02-22T21:36:23.643 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:36:23.643 INFO:teuthology.orchestra.run.trial026.stdout:[mon] 2026-02-22T21:36:23.643 INFO:teuthology.orchestra.run.trial026.stdout: debug ms = 1 2026-02-22T21:36:23.643 INFO:teuthology.orchestra.run.trial026.stdout: debug mon = 20 2026-02-22T21:36:23.643 INFO:teuthology.orchestra.run.trial026.stdout: debug paxos = 20 2026-02-22T21:36:23.644 INFO:teuthology.orchestra.run.trial026.stdout: debug auth = 20 2026-02-22T21:36:23.644 INFO:teuthology.orchestra.run.trial026.stdout: mon data avail warn = 5 2026-02-22T21:36:23.644 INFO:teuthology.orchestra.run.trial026.stdout: mon mgr mkfs grace = 240 2026-02-22T21:36:23.644 INFO:teuthology.orchestra.run.trial026.stdout: mon reweight min pgs per osd = 4 2026-02-22T21:36:23.644 INFO:teuthology.orchestra.run.trial026.stdout: mon osd reporter subtree level = osd 2026-02-22T21:36:23.644 INFO:teuthology.orchestra.run.trial026.stdout: mon osd prime pg temp = true 2026-02-22T21:36:23.644 INFO:teuthology.orchestra.run.trial026.stdout: mon reweight min bytes per osd = 10 2026-02-22T21:36:23.644 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:36:23.644 INFO:teuthology.orchestra.run.trial026.stdout: # rotate auth tickets quickly to exercise renewal paths 2026-02-22T21:36:23.644 INFO:teuthology.orchestra.run.trial026.stdout: auth mon ticket ttl = 660 # 11m 2026-02-22T21:36:23.644 INFO:teuthology.orchestra.run.trial026.stdout: auth service ticket ttl = 240 # 4m 2026-02-22T21:36:23.644 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:36:23.644 INFO:teuthology.orchestra.run.trial026.stdout: # don't complain about insecure global_id in the test suite 2026-02-22T21:36:23.645 INFO:teuthology.orchestra.run.trial026.stdout: mon_warn_on_insecure_global_id_reclaim = false 2026-02-22T21:36:23.645 INFO:teuthology.orchestra.run.trial026.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2026-02-22T21:36:23.645 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:36:23.645 INFO:teuthology.orchestra.run.trial026.stdout: # 1m isn't quite enough 2026-02-22T21:36:23.645 INFO:teuthology.orchestra.run.trial026.stdout: mon_down_mkfs_grace = 2m 2026-02-22T21:36:23.645 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:36:23.645 INFO:teuthology.orchestra.run.trial026.stdout: mon_warn_on_filestore_osds = false 2026-02-22T21:36:23.645 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:36:23.645 INFO:teuthology.orchestra.run.trial026.stdout:[client] 2026-02-22T21:36:23.645 INFO:teuthology.orchestra.run.trial026.stdout: rgw cache enabled = true 2026-02-22T21:36:23.645 INFO:teuthology.orchestra.run.trial026.stdout: rgw enable ops log = true 2026-02-22T21:36:23.645 INFO:teuthology.orchestra.run.trial026.stdout: rgw enable usage log = true 2026-02-22T21:36:23.646 INFO:teuthology.orchestra.run.trial026.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2026-02-22T21:36:23.646 INFO:teuthology.orchestra.run.trial026.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2026-02-22T21:36:23.646 INFO:teuthology.orchestra.run.trial026.stdout: debug rbd = 20 2026-02-22T21:36:23.646 INFO:teuthology.orchestra.run.trial026.stdout: debug rbd_mirror = 15 2026-02-22T21:36:23.646 INFO:teuthology.orchestra.run.trial026.stdout: log to stderr = False 2026-02-22T21:36:23.646 INFO:teuthology.orchestra.run.trial026.stdout: rbd default features = 125 2026-02-22T21:36:23.646 INFO:teuthology.orchestra.run.trial026.stdout: rbd mirror image policy type = simple 2026-02-22T21:36:23.646 INFO:teuthology.orchestra.run.trial026.stdout:[mon.a] 2026-02-22T21:36:23.646 INFO:teuthology.orchestra.run.trial026.stdout:[client.mirror.0] 2026-02-22T21:36:23.646 INFO:teuthology.orchestra.run.trial026.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T21:36:23.646 INFO:teuthology.orchestra.run.trial026.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T21:36:23.646 INFO:teuthology.orchestra.run.trial026.stdout:[client.mirror.1] 2026-02-22T21:36:23.646 INFO:teuthology.orchestra.run.trial026.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T21:36:23.647 INFO:teuthology.orchestra.run.trial026.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T21:36:23.647 INFO:teuthology.orchestra.run.trial026.stdout:[client.mirror.2] 2026-02-22T21:36:23.647 INFO:teuthology.orchestra.run.trial026.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T21:36:23.647 INFO:teuthology.orchestra.run.trial026.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T21:36:23.647 INFO:teuthology.orchestra.run.trial026.stdout:[client.mirror.3] 2026-02-22T21:36:23.647 INFO:teuthology.orchestra.run.trial026.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T21:36:23.647 INFO:teuthology.orchestra.run.trial026.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T21:36:23.648 INFO:tasks.ceph:Creating admin key on cluster2.mon.a... 2026-02-22T21:36:23.648 DEBUG:teuthology.orchestra.run.trial026:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *' /etc/ceph/cluster2.keyring 2026-02-22T21:36:23.728 INFO:tasks.ceph:Copying monmap to all nodes... 2026-02-22T21:36:23.729 DEBUG:teuthology.orchestra.run.trial026:> set -ex 2026-02-22T21:36:23.729 DEBUG:teuthology.orchestra.run.trial026:> dd if=/etc/ceph/cluster2.keyring of=/dev/stdout 2026-02-22T21:36:23.785 DEBUG:teuthology.orchestra.run.trial026:> set -ex 2026-02-22T21:36:23.785 DEBUG:teuthology.orchestra.run.trial026:> dd if=/home/ubuntu/cephtest/cluster2.monmap of=/dev/stdout 2026-02-22T21:36:23.842 INFO:tasks.ceph:Sending monmap to node ubuntu@trial004.front.sepia.ceph.com 2026-02-22T21:36:23.842 DEBUG:teuthology.orchestra.run.trial004:> set -ex 2026-02-22T21:36:23.842 DEBUG:teuthology.orchestra.run.trial004:> sudo dd of=/etc/ceph/cluster2.keyring 2026-02-22T21:36:23.842 DEBUG:teuthology.orchestra.run.trial004:> sudo chmod 0644 /etc/ceph/cluster2.keyring 2026-02-22T21:36:23.877 DEBUG:teuthology.orchestra.run.trial004:> set -ex 2026-02-22T21:36:23.877 DEBUG:teuthology.orchestra.run.trial004:> dd of=/home/ubuntu/cephtest/cluster2.monmap 2026-02-22T21:36:23.932 INFO:tasks.ceph:Sending monmap to node ubuntu@trial026.front.sepia.ceph.com 2026-02-22T21:36:23.933 DEBUG:teuthology.orchestra.run.trial026:> set -ex 2026-02-22T21:36:23.933 DEBUG:teuthology.orchestra.run.trial026:> sudo dd of=/etc/ceph/cluster2.keyring 2026-02-22T21:36:23.933 DEBUG:teuthology.orchestra.run.trial026:> sudo chmod 0644 /etc/ceph/cluster2.keyring 2026-02-22T21:36:23.966 DEBUG:teuthology.orchestra.run.trial026:> set -ex 2026-02-22T21:36:23.966 DEBUG:teuthology.orchestra.run.trial026:> dd of=/home/ubuntu/cephtest/cluster2.monmap 2026-02-22T21:36:24.023 INFO:tasks.ceph:Setting up mon nodes... 2026-02-22T21:36:24.023 INFO:tasks.ceph:Setting up mgr nodes... 2026-02-22T21:36:24.024 DEBUG:teuthology.orchestra.run.trial004:> sudo mkdir -p /var/lib/ceph/mgr/cluster2-x && sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=mgr.x /var/lib/ceph/mgr/cluster2-x/keyring 2026-02-22T21:36:24.066 INFO:teuthology.orchestra.run.trial004.stdout:creating /var/lib/ceph/mgr/cluster2-x/keyring 2026-02-22T21:36:24.068 INFO:tasks.ceph:Setting up mds nodes... 2026-02-22T21:36:24.069 INFO:tasks.ceph_client:Setting up client nodes... 2026-02-22T21:36:24.069 DEBUG:teuthology.orchestra.run.trial004:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.0 /etc/ceph/cluster2.client.0.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.0.keyring 2026-02-22T21:36:24.143 INFO:teuthology.orchestra.run.trial004.stdout:creating /etc/ceph/cluster2.client.0.keyring 2026-02-22T21:36:24.154 DEBUG:teuthology.orchestra.run.trial026:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror /etc/ceph/cluster2.client.mirror.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.keyring 2026-02-22T21:36:24.189 INFO:teuthology.orchestra.run.trial026.stdout:creating /etc/ceph/cluster2.client.mirror.keyring 2026-02-22T21:36:24.201 DEBUG:teuthology.orchestra.run.trial026:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.0 /etc/ceph/cluster2.client.mirror.0.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.0.keyring 2026-02-22T21:36:24.276 INFO:teuthology.orchestra.run.trial026.stdout:creating /etc/ceph/cluster2.client.mirror.0.keyring 2026-02-22T21:36:24.287 DEBUG:teuthology.orchestra.run.trial026:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.1 /etc/ceph/cluster2.client.mirror.1.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.1.keyring 2026-02-22T21:36:24.360 INFO:teuthology.orchestra.run.trial026.stdout:creating /etc/ceph/cluster2.client.mirror.1.keyring 2026-02-22T21:36:24.372 DEBUG:teuthology.orchestra.run.trial026:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.2 /etc/ceph/cluster2.client.mirror.2.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.2.keyring 2026-02-22T21:36:24.448 INFO:teuthology.orchestra.run.trial026.stdout:creating /etc/ceph/cluster2.client.mirror.2.keyring 2026-02-22T21:36:24.458 DEBUG:teuthology.orchestra.run.trial026:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.3 /etc/ceph/cluster2.client.mirror.3.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.3.keyring 2026-02-22T21:36:24.532 INFO:teuthology.orchestra.run.trial026.stdout:creating /etc/ceph/cluster2.client.mirror.3.keyring 2026-02-22T21:36:24.543 DEBUG:teuthology.orchestra.run.trial026:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.4 /etc/ceph/cluster2.client.mirror.4.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.4.keyring 2026-02-22T21:36:24.616 INFO:teuthology.orchestra.run.trial026.stdout:creating /etc/ceph/cluster2.client.mirror.4.keyring 2026-02-22T21:36:24.627 DEBUG:teuthology.orchestra.run.trial026:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.5 /etc/ceph/cluster2.client.mirror.5.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.5.keyring 2026-02-22T21:36:24.700 INFO:teuthology.orchestra.run.trial026.stdout:creating /etc/ceph/cluster2.client.mirror.5.keyring 2026-02-22T21:36:24.710 DEBUG:teuthology.orchestra.run.trial026:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.6 /etc/ceph/cluster2.client.mirror.6.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.6.keyring 2026-02-22T21:36:24.785 INFO:teuthology.orchestra.run.trial026.stdout:creating /etc/ceph/cluster2.client.mirror.6.keyring 2026-02-22T21:36:24.795 INFO:tasks.ceph:Running mkfs on osd nodes... 2026-02-22T21:36:24.795 INFO:tasks.ceph:ctx.disk_config.remote_to_roles_to_dev: {Remote(name='ubuntu@trial004.front.sepia.ceph.com'): {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'}, Remote(name='ubuntu@trial026.front.sepia.ceph.com'): {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'}} 2026-02-22T21:36:24.795 DEBUG:teuthology.orchestra.run.trial026:> sudo mkdir -p /var/lib/ceph/osd/cluster2-0 2026-02-22T21:36:24.859 INFO:tasks.ceph:roles_to_devs: {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'} 2026-02-22T21:36:24.859 INFO:tasks.ceph:role: cluster2.osd.0 2026-02-22T21:36:24.859 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_1 on ubuntu@trial026.front.sepia.ceph.com 2026-02-22T21:36:24.860 DEBUG:teuthology.orchestra.run.trial026:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_1 2026-02-22T21:36:24.923 INFO:teuthology.orchestra.run.trial026.stdout:meta-data=/dev/vg_nvme/lv_1 isize=2048 agcount=4, agsize=45875200 blks 2026-02-22T21:36:24.923 INFO:teuthology.orchestra.run.trial026.stdout: = sectsz=4096 attr=2, projid32bit=1 2026-02-22T21:36:24.923 INFO:teuthology.orchestra.run.trial026.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2026-02-22T21:36:24.923 INFO:teuthology.orchestra.run.trial026.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2026-02-22T21:36:24.923 INFO:teuthology.orchestra.run.trial026.stdout:data = bsize=4096 blocks=183500800, imaxpct=25 2026-02-22T21:36:24.923 INFO:teuthology.orchestra.run.trial026.stdout: = sunit=0 swidth=0 blks 2026-02-22T21:36:24.923 INFO:teuthology.orchestra.run.trial026.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2026-02-22T21:36:24.923 INFO:teuthology.orchestra.run.trial026.stdout:log =internal log bsize=4096 blocks=89600, version=2 2026-02-22T21:36:24.923 INFO:teuthology.orchestra.run.trial026.stdout: = sectsz=4096 sunit=1 blks, lazy-count=1 2026-02-22T21:36:24.923 INFO:teuthology.orchestra.run.trial026.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2026-02-22T21:36:25.490 INFO:teuthology.orchestra.run.trial026.stdout:Discarding blocks...Done. 2026-02-22T21:36:25.492 INFO:tasks.ceph:mount /dev/vg_nvme/lv_1 on ubuntu@trial026.front.sepia.ceph.com -o noatime 2026-02-22T21:36:25.492 DEBUG:teuthology.orchestra.run.trial026:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_1 /var/lib/ceph/osd/cluster2-0 2026-02-22T21:36:25.575 DEBUG:teuthology.orchestra.run.trial026:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster2-0 2026-02-22T21:36:25.646 DEBUG:teuthology.orchestra.run.trial026:> sudo mkdir -p /var/lib/ceph/osd/cluster2-1 2026-02-22T21:36:25.710 INFO:tasks.ceph:roles_to_devs: {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'} 2026-02-22T21:36:25.711 INFO:tasks.ceph:role: cluster2.osd.1 2026-02-22T21:36:25.711 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_2 on ubuntu@trial026.front.sepia.ceph.com 2026-02-22T21:36:25.711 DEBUG:teuthology.orchestra.run.trial026:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_2 2026-02-22T21:36:25.776 INFO:teuthology.orchestra.run.trial026.stdout:meta-data=/dev/vg_nvme/lv_2 isize=2048 agcount=4, agsize=45875200 blks 2026-02-22T21:36:25.776 INFO:teuthology.orchestra.run.trial026.stdout: = sectsz=4096 attr=2, projid32bit=1 2026-02-22T21:36:25.777 INFO:teuthology.orchestra.run.trial026.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2026-02-22T21:36:25.777 INFO:teuthology.orchestra.run.trial026.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2026-02-22T21:36:25.777 INFO:teuthology.orchestra.run.trial026.stdout:data = bsize=4096 blocks=183500800, imaxpct=25 2026-02-22T21:36:25.777 INFO:teuthology.orchestra.run.trial026.stdout: = sunit=0 swidth=0 blks 2026-02-22T21:36:25.777 INFO:teuthology.orchestra.run.trial026.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2026-02-22T21:36:25.777 INFO:teuthology.orchestra.run.trial026.stdout:log =internal log bsize=4096 blocks=89600, version=2 2026-02-22T21:36:25.777 INFO:teuthology.orchestra.run.trial026.stdout: = sectsz=4096 sunit=1 blks, lazy-count=1 2026-02-22T21:36:25.777 INFO:teuthology.orchestra.run.trial026.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2026-02-22T21:36:26.343 INFO:teuthology.orchestra.run.trial026.stdout:Discarding blocks...Done. 2026-02-22T21:36:26.345 INFO:tasks.ceph:mount /dev/vg_nvme/lv_2 on ubuntu@trial026.front.sepia.ceph.com -o noatime 2026-02-22T21:36:26.346 DEBUG:teuthology.orchestra.run.trial026:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_2 /var/lib/ceph/osd/cluster2-1 2026-02-22T21:36:26.418 DEBUG:teuthology.orchestra.run.trial026:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster2-1 2026-02-22T21:36:26.487 DEBUG:teuthology.orchestra.run.trial026:> sudo mkdir -p /var/lib/ceph/osd/cluster2-2 2026-02-22T21:36:26.553 INFO:tasks.ceph:roles_to_devs: {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'} 2026-02-22T21:36:26.553 INFO:tasks.ceph:role: cluster2.osd.2 2026-02-22T21:36:26.553 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_3 on ubuntu@trial026.front.sepia.ceph.com 2026-02-22T21:36:26.553 DEBUG:teuthology.orchestra.run.trial026:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_3 2026-02-22T21:36:26.618 INFO:teuthology.orchestra.run.trial026.stdout:meta-data=/dev/vg_nvme/lv_3 isize=2048 agcount=4, agsize=45875200 blks 2026-02-22T21:36:26.618 INFO:teuthology.orchestra.run.trial026.stdout: = sectsz=4096 attr=2, projid32bit=1 2026-02-22T21:36:26.619 INFO:teuthology.orchestra.run.trial026.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2026-02-22T21:36:26.619 INFO:teuthology.orchestra.run.trial026.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2026-02-22T21:36:26.619 INFO:teuthology.orchestra.run.trial026.stdout:data = bsize=4096 blocks=183500800, imaxpct=25 2026-02-22T21:36:26.619 INFO:teuthology.orchestra.run.trial026.stdout: = sunit=0 swidth=0 blks 2026-02-22T21:36:26.619 INFO:teuthology.orchestra.run.trial026.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2026-02-22T21:36:26.619 INFO:teuthology.orchestra.run.trial026.stdout:log =internal log bsize=4096 blocks=89600, version=2 2026-02-22T21:36:26.619 INFO:teuthology.orchestra.run.trial026.stdout: = sectsz=4096 sunit=1 blks, lazy-count=1 2026-02-22T21:36:26.619 INFO:teuthology.orchestra.run.trial026.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2026-02-22T21:36:27.174 INFO:teuthology.orchestra.run.trial026.stdout:Discarding blocks...Done. 2026-02-22T21:36:27.176 INFO:tasks.ceph:mount /dev/vg_nvme/lv_3 on ubuntu@trial026.front.sepia.ceph.com -o noatime 2026-02-22T21:36:27.176 DEBUG:teuthology.orchestra.run.trial026:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_3 /var/lib/ceph/osd/cluster2-2 2026-02-22T21:36:27.243 DEBUG:teuthology.orchestra.run.trial026:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster2-2 2026-02-22T21:36:27.310 DEBUG:teuthology.orchestra.run.trial026:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster2 --mkfs --mkkey -i 0 --monmap /home/ubuntu/cephtest/cluster2.monmap 2026-02-22T21:36:27.389 INFO:teuthology.orchestra.run.trial026.stderr:2026-02-22T21:36:27.387+0000 7f73652ba780 -1 auth: error reading file: /var/lib/ceph/osd/cluster2-0/keyring: can't open /var/lib/ceph/osd/cluster2-0/keyring: (2) No such file or directory 2026-02-22T21:36:27.390 INFO:teuthology.orchestra.run.trial026.stderr:2026-02-22T21:36:27.387+0000 7f73652ba780 -1 created new key in keyring /var/lib/ceph/osd/cluster2-0/keyring 2026-02-22T21:36:27.390 INFO:teuthology.orchestra.run.trial026.stderr:2026-02-22T21:36:27.387+0000 7f73652ba780 -1 bdev(0x55f091d23800 /var/lib/ceph/osd/cluster2-0/block) open stat got: (1) Operation not permitted 2026-02-22T21:36:27.390 INFO:teuthology.orchestra.run.trial026.stderr:2026-02-22T21:36:27.387+0000 7f73652ba780 -1 bluestore(/var/lib/ceph/osd/cluster2-0) _read_fsid unparsable uuid 2026-02-22T21:36:30.284 DEBUG:teuthology.orchestra.run.trial026:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster2-0 2026-02-22T21:36:30.351 DEBUG:teuthology.orchestra.run.trial026:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster2 --mkfs --mkkey -i 1 --monmap /home/ubuntu/cephtest/cluster2.monmap 2026-02-22T21:36:30.428 INFO:teuthology.orchestra.run.trial026.stderr:2026-02-22T21:36:30.426+0000 7f6245610780 -1 auth: error reading file: /var/lib/ceph/osd/cluster2-1/keyring: can't open /var/lib/ceph/osd/cluster2-1/keyring: (2) No such file or directory 2026-02-22T21:36:30.429 INFO:teuthology.orchestra.run.trial026.stderr:2026-02-22T21:36:30.426+0000 7f6245610780 -1 created new key in keyring /var/lib/ceph/osd/cluster2-1/keyring 2026-02-22T21:36:30.429 INFO:teuthology.orchestra.run.trial026.stderr:2026-02-22T21:36:30.426+0000 7f6245610780 -1 bdev(0x56244ff99800 /var/lib/ceph/osd/cluster2-1/block) open stat got: (1) Operation not permitted 2026-02-22T21:36:30.429 INFO:teuthology.orchestra.run.trial026.stderr:2026-02-22T21:36:30.426+0000 7f6245610780 -1 bluestore(/var/lib/ceph/osd/cluster2-1) _read_fsid unparsable uuid 2026-02-22T21:36:33.821 DEBUG:teuthology.orchestra.run.trial026:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster2-1 2026-02-22T21:36:33.886 DEBUG:teuthology.orchestra.run.trial026:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster2 --mkfs --mkkey -i 2 --monmap /home/ubuntu/cephtest/cluster2.monmap 2026-02-22T21:36:33.964 INFO:teuthology.orchestra.run.trial026.stderr:2026-02-22T21:36:33.962+0000 7f7748621780 -1 auth: error reading file: /var/lib/ceph/osd/cluster2-2/keyring: can't open /var/lib/ceph/osd/cluster2-2/keyring: (2) No such file or directory 2026-02-22T21:36:33.964 INFO:teuthology.orchestra.run.trial026.stderr:2026-02-22T21:36:33.962+0000 7f7748621780 -1 created new key in keyring /var/lib/ceph/osd/cluster2-2/keyring 2026-02-22T21:36:33.964 INFO:teuthology.orchestra.run.trial026.stderr:2026-02-22T21:36:33.962+0000 7f7748621780 -1 bdev(0x55b5fed3b800 /var/lib/ceph/osd/cluster2-2/block) open stat got: (1) Operation not permitted 2026-02-22T21:36:33.964 INFO:teuthology.orchestra.run.trial026.stderr:2026-02-22T21:36:33.962+0000 7f7748621780 -1 bluestore(/var/lib/ceph/osd/cluster2-2) _read_fsid unparsable uuid 2026-02-22T21:36:36.864 DEBUG:teuthology.orchestra.run.trial026:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster2-2 2026-02-22T21:36:36.928 INFO:tasks.ceph:Reading keys from all nodes... 2026-02-22T21:36:36.928 DEBUG:teuthology.orchestra.run.trial004:> set -ex 2026-02-22T21:36:36.928 DEBUG:teuthology.orchestra.run.trial004:> sudo dd if=/var/lib/ceph/mgr/cluster2-x/keyring of=/dev/stdout 2026-02-22T21:36:36.956 DEBUG:teuthology.orchestra.run.trial026:> set -ex 2026-02-22T21:36:36.956 DEBUG:teuthology.orchestra.run.trial026:> sudo dd if=/var/lib/ceph/osd/cluster2-0/keyring of=/dev/stdout 2026-02-22T21:36:36.994 DEBUG:teuthology.orchestra.run.trial026:> set -ex 2026-02-22T21:36:36.995 DEBUG:teuthology.orchestra.run.trial026:> sudo dd if=/var/lib/ceph/osd/cluster2-1/keyring of=/dev/stdout 2026-02-22T21:36:37.058 DEBUG:teuthology.orchestra.run.trial026:> set -ex 2026-02-22T21:36:37.058 DEBUG:teuthology.orchestra.run.trial026:> sudo dd if=/var/lib/ceph/osd/cluster2-2/keyring of=/dev/stdout 2026-02-22T21:36:37.122 DEBUG:teuthology.orchestra.run.trial004:> set -ex 2026-02-22T21:36:37.122 DEBUG:teuthology.orchestra.run.trial004:> dd if=/etc/ceph/cluster2.client.0.keyring of=/dev/stdout 2026-02-22T21:36:37.139 DEBUG:teuthology.orchestra.run.trial026:> set -ex 2026-02-22T21:36:37.139 DEBUG:teuthology.orchestra.run.trial026:> dd if=/etc/ceph/cluster2.client.mirror.keyring of=/dev/stdout 2026-02-22T21:36:37.179 DEBUG:teuthology.orchestra.run.trial026:> set -ex 2026-02-22T21:36:37.179 DEBUG:teuthology.orchestra.run.trial026:> dd if=/etc/ceph/cluster2.client.mirror.0.keyring of=/dev/stdout 2026-02-22T21:36:37.236 DEBUG:teuthology.orchestra.run.trial026:> set -ex 2026-02-22T21:36:37.236 DEBUG:teuthology.orchestra.run.trial026:> dd if=/etc/ceph/cluster2.client.mirror.1.keyring of=/dev/stdout 2026-02-22T21:36:37.292 DEBUG:teuthology.orchestra.run.trial026:> set -ex 2026-02-22T21:36:37.293 DEBUG:teuthology.orchestra.run.trial026:> dd if=/etc/ceph/cluster2.client.mirror.2.keyring of=/dev/stdout 2026-02-22T21:36:37.348 DEBUG:teuthology.orchestra.run.trial026:> set -ex 2026-02-22T21:36:37.348 DEBUG:teuthology.orchestra.run.trial026:> dd if=/etc/ceph/cluster2.client.mirror.3.keyring of=/dev/stdout 2026-02-22T21:36:37.404 DEBUG:teuthology.orchestra.run.trial026:> set -ex 2026-02-22T21:36:37.404 DEBUG:teuthology.orchestra.run.trial026:> dd if=/etc/ceph/cluster2.client.mirror.4.keyring of=/dev/stdout 2026-02-22T21:36:37.460 DEBUG:teuthology.orchestra.run.trial026:> set -ex 2026-02-22T21:36:37.460 DEBUG:teuthology.orchestra.run.trial026:> dd if=/etc/ceph/cluster2.client.mirror.5.keyring of=/dev/stdout 2026-02-22T21:36:37.516 DEBUG:teuthology.orchestra.run.trial026:> set -ex 2026-02-22T21:36:37.517 DEBUG:teuthology.orchestra.run.trial026:> dd if=/etc/ceph/cluster2.client.mirror.6.keyring of=/dev/stdout 2026-02-22T21:36:37.574 INFO:tasks.ceph:Adding keys to all mons... 2026-02-22T21:36:37.574 DEBUG:teuthology.orchestra.run.trial026:> sudo tee -a /etc/ceph/cluster2.keyring 2026-02-22T21:36:37.638 INFO:teuthology.orchestra.run.trial026.stdout:[mgr.x] 2026-02-22T21:36:37.638 INFO:teuthology.orchestra.run.trial026.stdout: key = AQDYdptpT7XxAxAAUgMH2yf++/6m9u4lzXS4aQ== 2026-02-22T21:36:37.638 INFO:teuthology.orchestra.run.trial026.stdout:[osd.0] 2026-02-22T21:36:37.639 INFO:teuthology.orchestra.run.trial026.stdout: key = AQDbdptp67IhFxAAV486XtyhNXGD42RMRngoYA== 2026-02-22T21:36:37.639 INFO:teuthology.orchestra.run.trial026.stdout:[osd.1] 2026-02-22T21:36:37.639 INFO:teuthology.orchestra.run.trial026.stdout: key = AQDedptpC414GRAAC69gKnEcEZnhgMdTQalBLg== 2026-02-22T21:36:37.639 INFO:teuthology.orchestra.run.trial026.stdout:[osd.2] 2026-02-22T21:36:37.639 INFO:teuthology.orchestra.run.trial026.stdout: key = AQDhdptp7BNoORAAVuRU87v8Q7G/bkdmB/XIGw== 2026-02-22T21:36:37.639 INFO:teuthology.orchestra.run.trial026.stdout:[client.0] 2026-02-22T21:36:37.639 INFO:teuthology.orchestra.run.trial026.stdout: key = AQDYdptpFZuFCBAAIdbJ+ZrSGEYRq43f0B6rIw== 2026-02-22T21:36:37.639 INFO:teuthology.orchestra.run.trial026.stdout:[client.mirror] 2026-02-22T21:36:37.639 INFO:teuthology.orchestra.run.trial026.stdout: key = AQDYdptpgzY6CxAAQJtwbthKMKzokmnKMHmSnQ== 2026-02-22T21:36:37.639 INFO:teuthology.orchestra.run.trial026.stdout:[client.mirror.0] 2026-02-22T21:36:37.639 INFO:teuthology.orchestra.run.trial026.stdout: key = AQDYdptpSjloEBAAzbY3aIIsPqDMUlGlyoJU3w== 2026-02-22T21:36:37.639 INFO:teuthology.orchestra.run.trial026.stdout:[client.mirror.1] 2026-02-22T21:36:37.639 INFO:teuthology.orchestra.run.trial026.stdout: key = AQDYdptpCRdzFRAAsLYLN5bi/C8V+ae6X66s9w== 2026-02-22T21:36:37.639 INFO:teuthology.orchestra.run.trial026.stdout:[client.mirror.2] 2026-02-22T21:36:37.639 INFO:teuthology.orchestra.run.trial026.stdout: key = AQDYdptpSIqlGhAAh1xP+oEdKaG+ueEzQh82BA== 2026-02-22T21:36:37.639 INFO:teuthology.orchestra.run.trial026.stdout:[client.mirror.3] 2026-02-22T21:36:37.639 INFO:teuthology.orchestra.run.trial026.stdout: key = AQDYdptpJtKlHxAAqCaG2hbinah7q3snnYlvhQ== 2026-02-22T21:36:37.639 INFO:teuthology.orchestra.run.trial026.stdout:[client.mirror.4] 2026-02-22T21:36:37.640 INFO:teuthology.orchestra.run.trial026.stdout: key = AQDYdptpXdexJBAASK47OjMcrFWxXRL8gAyWNw== 2026-02-22T21:36:37.640 INFO:teuthology.orchestra.run.trial026.stdout:[client.mirror.5] 2026-02-22T21:36:37.640 INFO:teuthology.orchestra.run.trial026.stdout: key = AQDYdptpJ0SzKRAA3ZJbXNOf0pDJls+EOQujuw== 2026-02-22T21:36:37.640 INFO:teuthology.orchestra.run.trial026.stdout:[client.mirror.6] 2026-02-22T21:36:37.640 INFO:teuthology.orchestra.run.trial026.stdout: key = AQDYdptp5mPDLhAAMMi2bPHyt1enEIgRLrPP8A== 2026-02-22T21:36:37.641 DEBUG:teuthology.orchestra.run.trial026:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=mgr.x --cap mon 'allow profile mgr' --cap osd 'allow *' --cap mds 'allow *' 2026-02-22T21:36:37.720 DEBUG:teuthology.orchestra.run.trial026:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=osd.0 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2026-02-22T21:36:37.800 DEBUG:teuthology.orchestra.run.trial026:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=osd.1 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2026-02-22T21:36:37.879 DEBUG:teuthology.orchestra.run.trial026:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=osd.2 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2026-02-22T21:36:37.960 DEBUG:teuthology.orchestra.run.trial026:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.0 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T21:36:38.042 DEBUG:teuthology.orchestra.run.trial026:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T21:36:38.123 DEBUG:teuthology.orchestra.run.trial026:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.0 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T21:36:38.204 DEBUG:teuthology.orchestra.run.trial026:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.1 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T21:36:38.285 DEBUG:teuthology.orchestra.run.trial026:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.2 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T21:36:38.367 DEBUG:teuthology.orchestra.run.trial026:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.3 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T21:36:38.449 DEBUG:teuthology.orchestra.run.trial026:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.4 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T21:36:38.529 DEBUG:teuthology.orchestra.run.trial026:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.5 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T21:36:38.610 DEBUG:teuthology.orchestra.run.trial026:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.6 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T21:36:38.691 INFO:tasks.ceph:Running mkfs on mon nodes... 2026-02-22T21:36:38.692 DEBUG:teuthology.orchestra.run.trial026:> sudo mkdir -p /var/lib/ceph/mon/cluster2-a 2026-02-22T21:36:38.757 DEBUG:teuthology.orchestra.run.trial026:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-mon --cluster cluster2 --mkfs -i a --monmap /home/ubuntu/cephtest/cluster2.monmap --keyring /etc/ceph/cluster2.keyring 2026-02-22T21:36:38.840 DEBUG:teuthology.orchestra.run.trial026:> sudo chown -R ceph:ceph /var/lib/ceph/mon/cluster2-a 2026-02-22T21:36:38.904 DEBUG:teuthology.orchestra.run.trial026:> rm -- /home/ubuntu/cephtest/cluster2.monmap 2026-02-22T21:36:38.960 INFO:tasks.ceph:Starting mon daemons in cluster cluster2... 2026-02-22T21:36:38.960 INFO:tasks.ceph.cluster2.mon.a:Restarting daemon 2026-02-22T21:36:38.961 DEBUG:teuthology.orchestra.run.trial026:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mon -f --cluster cluster2 -i a 2026-02-22T21:36:39.002 INFO:tasks.ceph.cluster2.mon.a:Started 2026-02-22T21:36:39.002 INFO:tasks.ceph:Starting mgr daemons in cluster cluster2... 2026-02-22T21:36:39.003 INFO:tasks.ceph.cluster2.mgr.x:Restarting daemon 2026-02-22T21:36:39.003 DEBUG:teuthology.orchestra.run.trial004:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mgr -f --cluster cluster2 -i x 2026-02-22T21:36:39.006 INFO:tasks.ceph.cluster2.mgr.x:Started 2026-02-22T21:36:39.006 INFO:tasks.ceph:Setting crush tunables to default 2026-02-22T21:36:39.006 DEBUG:teuthology.orchestra.run.trial026:> sudo ceph --cluster cluster2 osd crush tunables default 2026-02-22T21:36:39.108 INFO:teuthology.orchestra.run.trial026.stderr:adjusted tunables profile to default 2026-02-22T21:36:39.115 INFO:tasks.ceph:check_enable_crimson: False 2026-02-22T21:36:39.116 INFO:tasks.ceph:Starting osd daemons in cluster cluster2... 2026-02-22T21:36:39.116 DEBUG:teuthology.orchestra.run.trial026:> set -ex 2026-02-22T21:36:39.116 DEBUG:teuthology.orchestra.run.trial026:> sudo dd if=/var/lib/ceph/osd/cluster2-0/fsid of=/dev/stdout 2026-02-22T21:36:39.140 DEBUG:teuthology.orchestra.run.trial026:> set -ex 2026-02-22T21:36:39.140 DEBUG:teuthology.orchestra.run.trial026:> sudo dd if=/var/lib/ceph/osd/cluster2-1/fsid of=/dev/stdout 2026-02-22T21:36:39.203 DEBUG:teuthology.orchestra.run.trial026:> set -ex 2026-02-22T21:36:39.204 DEBUG:teuthology.orchestra.run.trial026:> sudo dd if=/var/lib/ceph/osd/cluster2-2/fsid of=/dev/stdout 2026-02-22T21:36:39.268 DEBUG:teuthology.orchestra.run.trial026:> sudo ceph --cluster cluster2 osd new 5ebb4bdc-b6e6-4276-a34c-a27c8810395c 0 2026-02-22T21:36:39.342 INFO:tasks.ceph.cluster2.mgr.x.trial004.stderr:2026-02-22T21:36:39.340+0000 7fe0f4a2d140 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2026-02-22T21:36:39.410 INFO:teuthology.orchestra.run.trial026.stdout:0 2026-02-22T21:36:39.418 DEBUG:teuthology.orchestra.run.trial026:> sudo ceph --cluster cluster2 osd new 24ae9b1a-c77d-483c-abb8-148e6becaaca 1 2026-02-22T21:36:39.458 INFO:tasks.ceph.cluster2.mgr.x.trial004.stderr:2026-02-22T21:36:39.458+0000 7fe0f4a2d140 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2026-02-22T21:36:39.480 INFO:tasks.ceph.cluster2.mgr.x.trial004.stderr:2026-02-22T21:36:39.480+0000 7fe0f4a2d140 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2026-02-22T21:36:39.506 INFO:tasks.ceph.cluster2.mgr.x.trial004.stderr:2026-02-22T21:36:39.505+0000 7fe0f4a2d140 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2026-02-22T21:36:39.520 INFO:teuthology.orchestra.run.trial026.stdout:1 2026-02-22T21:36:39.527 DEBUG:teuthology.orchestra.run.trial026:> sudo ceph --cluster cluster2 osd new d5ca9f56-e467-4298-8a36-49cf123cc629 2 2026-02-22T21:36:39.529 INFO:tasks.ceph.cluster2.mgr.x.trial004.stderr:2026-02-22T21:36:39.526+0000 7fe0f4a2d140 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2026-02-22T21:36:39.552 INFO:tasks.ceph.cluster2.mgr.x.trial004.stderr:2026-02-22T21:36:39.551+0000 7fe0f4a2d140 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2026-02-22T21:36:39.602 INFO:tasks.ceph.cluster2.mgr.x.trial004.stderr:2026-02-22T21:36:39.602+0000 7fe0f4a2d140 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2026-02-22T21:36:39.630 INFO:teuthology.orchestra.run.trial026.stdout:2 2026-02-22T21:36:39.638 INFO:tasks.ceph.cluster2.osd.0:Restarting daemon 2026-02-22T21:36:39.638 DEBUG:teuthology.orchestra.run.trial026:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster2 -i 0 2026-02-22T21:36:39.640 INFO:tasks.ceph.cluster2.osd.0:Started 2026-02-22T21:36:39.641 INFO:tasks.ceph.cluster2.osd.1:Restarting daemon 2026-02-22T21:36:39.641 DEBUG:teuthology.orchestra.run.trial026:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster2 -i 1 2026-02-22T21:36:39.643 INFO:tasks.ceph.cluster2.osd.1:Started 2026-02-22T21:36:39.644 INFO:tasks.ceph.cluster2.osd.2:Restarting daemon 2026-02-22T21:36:39.644 DEBUG:teuthology.orchestra.run.trial026:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster2 -i 2 2026-02-22T21:36:39.646 INFO:tasks.ceph.cluster2.osd.2:Started 2026-02-22T21:36:39.646 DEBUG:teuthology.orchestra.run.trial026:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd dump --format=json 2026-02-22T21:36:39.661 INFO:tasks.ceph.cluster2.mgr.x.trial004.stderr:/usr/lib64/python3.9/site-packages/scipy/__init__.py:73: UserWarning: NumPy was imported from a Python sub-interpreter but NumPy does not properly support sub-interpreters. This will likely work for most users but might cause hard to track down issues or subtle bugs. A common user of the rare sub-interpreter feature is wsgi which also allows single-interpreter mode. 2026-02-22T21:36:39.661 INFO:tasks.ceph.cluster2.mgr.x.trial004.stderr:Improvements in the case of bugs are welcome, but is not on the NumPy roadmap, and full support may require significant effort to achieve. 2026-02-22T21:36:39.661 INFO:tasks.ceph.cluster2.mgr.x.trial004.stderr: from numpy import show_config as show_numpy_config 2026-02-22T21:36:39.663 INFO:tasks.ceph.cluster2.mgr.x.trial004.stderr:2026-02-22T21:36:39.663+0000 7fe0f4a2d140 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2026-02-22T21:36:39.688 INFO:tasks.ceph.cluster2.mgr.x.trial004.stderr:2026-02-22T21:36:39.687+0000 7fe0f4a2d140 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2026-02-22T21:36:39.741 INFO:tasks.ceph.cluster2.mgr.x.trial004.stderr:2026-02-22T21:36:39.740+0000 7fe0f4a2d140 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2026-02-22T21:36:39.747 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:36:39.747 INFO:teuthology.orchestra.run.trial026.stdout:{"epoch":5,"fsid":"2e52b784-3eb2-4856-a6f6-868b97c4e845","created":"2026-02-22T21:36:39.062947+0000","modified":"2026-02-22T21:36:39.629207+0000","last_up_change":"0.000000","last_in_change":"2026-02-22T21:36:39.629207+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"5ebb4bdc-b6e6-4276-a34c-a27c8810395c","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"24ae9b1a-c77d-483c-abb8-148e6becaaca","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"d5ca9f56-e467-4298-8a36-49cf123cc629","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-22T21:36:39.754 INFO:tasks.ceph.ceph_manager.cluster2:[] 2026-02-22T21:36:39.755 INFO:tasks.ceph:Waiting for OSDs to come up 2026-02-22T21:36:39.781 INFO:tasks.ceph.cluster2.mgr.x.trial004.stderr:2026-02-22T21:36:39.781+0000 7fe0f4a2d140 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2026-02-22T21:36:39.805 INFO:tasks.ceph.cluster2.mgr.x.trial004.stderr:2026-02-22T21:36:39.805+0000 7fe0f4a2d140 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2026-02-22T21:36:39.828 INFO:tasks.ceph.cluster2.mgr.x.trial004.stderr:2026-02-22T21:36:39.828+0000 7fe0f4a2d140 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2026-02-22T21:36:39.852 INFO:tasks.ceph.cluster2.mgr.x.trial004.stderr:2026-02-22T21:36:39.852+0000 7fe0f4a2d140 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2026-02-22T21:36:39.872 INFO:tasks.ceph.cluster2.mgr.x.trial004.stderr:2026-02-22T21:36:39.872+0000 7fe0f4a2d140 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2026-02-22T21:36:40.024 INFO:tasks.ceph.cluster2.mgr.x.trial004.stderr:2026-02-22T21:36:40.023+0000 7fe0f4a2d140 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2026-02-22T21:36:40.057 DEBUG:teuthology.orchestra.run.trial026:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster2 osd dump --format=json 2026-02-22T21:36:40.079 INFO:tasks.ceph.cluster2.mgr.x.trial004.stderr:2026-02-22T21:36:40.077+0000 7fe0f4a2d140 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2026-02-22T21:36:40.100 INFO:tasks.ceph.cluster2.mgr.x.trial004.stderr:2026-02-22T21:36:40.098+0000 7fe0f4a2d140 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2026-02-22T21:36:40.121 INFO:tasks.ceph.cluster2.mgr.x.trial004.stderr:2026-02-22T21:36:40.120+0000 7fe0f4a2d140 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2026-02-22T21:36:40.153 INFO:teuthology.misc.health.trial026.stdout: 2026-02-22T21:36:40.153 INFO:teuthology.misc.health.trial026.stdout:{"epoch":5,"fsid":"2e52b784-3eb2-4856-a6f6-868b97c4e845","created":"2026-02-22T21:36:39.062947+0000","modified":"2026-02-22T21:36:39.629207+0000","last_up_change":"0.000000","last_in_change":"2026-02-22T21:36:39.629207+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"5ebb4bdc-b6e6-4276-a34c-a27c8810395c","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"24ae9b1a-c77d-483c-abb8-148e6becaaca","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"d5ca9f56-e467-4298-8a36-49cf123cc629","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-22T21:36:40.159 DEBUG:teuthology.misc:0 of 3 OSDs are up 2026-02-22T21:36:40.173 INFO:tasks.ceph.cluster2.mgr.x.trial004.stderr:2026-02-22T21:36:40.172+0000 7fe0f4a2d140 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2026-02-22T21:36:40.193 INFO:tasks.ceph.cluster2.mgr.x.trial004.stderr:2026-02-22T21:36:40.192+0000 7fe0f4a2d140 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2026-02-22T21:36:40.245 INFO:tasks.ceph.cluster2.mgr.x.trial004.stderr:2026-02-22T21:36:40.243+0000 7fe0f4a2d140 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2026-02-22T21:36:40.563 INFO:tasks.ceph.cluster2.mgr.x.trial004.stderr:2026-02-22T21:36:40.561+0000 7fe0f4a2d140 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2026-02-22T21:36:40.584 INFO:tasks.ceph.cluster2.mgr.x.trial004.stderr:2026-02-22T21:36:40.582+0000 7fe0f4a2d140 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2026-02-22T21:36:40.911 INFO:tasks.ceph.cluster2.mgr.x.trial004.stderr:2026-02-22T21:36:40.910+0000 7fe0f4a2d140 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2026-02-22T21:36:41.047 INFO:tasks.ceph.cluster2.osd.2.trial026.stderr:2026-02-22T21:36:41.045+0000 7f32f322c780 -1 Falling back to public interface 2026-02-22T21:36:41.537 INFO:tasks.ceph.cluster2.osd.0.trial026.stderr:2026-02-22T21:36:41.535+0000 7ff4c5c97780 -1 Falling back to public interface 2026-02-22T21:36:41.553 INFO:tasks.ceph.cluster2.osd.1.trial026.stderr:2026-02-22T21:36:41.551+0000 7f830c962780 -1 Falling back to public interface 2026-02-22T21:36:41.920 INFO:tasks.ceph.cluster2.mgr.x.trial004.stderr:2026-02-22T21:36:41.918+0000 7fe012167640 -1 mgr.server handle_report got status from non-daemon mon.a 2026-02-22T21:36:45.038 INFO:tasks.ceph.cluster2.osd.2.trial026.stderr:2026-02-22T21:36:45.036+0000 7f32f322c780 -1 osd.2 0 log_to_monitors true 2026-02-22T21:36:45.264 INFO:tasks.ceph.cluster2.osd.0.trial026.stderr:2026-02-22T21:36:45.262+0000 7ff4c5c97780 -1 osd.0 0 log_to_monitors true 2026-02-22T21:36:45.566 INFO:tasks.ceph.cluster2.osd.1.trial026.stderr:2026-02-22T21:36:45.564+0000 7f830c962780 -1 osd.1 0 log_to_monitors true 2026-02-22T21:36:46.104 INFO:tasks.ceph.cluster2.osd.2.trial026.stderr:2026-02-22T21:36:46.103+0000 7f32ef1b9640 -1 osd.2 0 waiting for initial osdmap 2026-02-22T21:36:46.106 INFO:tasks.ceph.cluster2.osd.2.trial026.stderr:2026-02-22T21:36:46.105+0000 7f32e6fdb640 -1 osd.2 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2026-02-22T21:36:46.462 DEBUG:teuthology.orchestra.run.trial026:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster2 osd dump --format=json 2026-02-22T21:36:46.610 INFO:teuthology.misc.health.trial026.stdout: 2026-02-22T21:36:46.610 INFO:teuthology.misc.health.trial026.stdout:{"epoch":7,"fsid":"2e52b784-3eb2-4856-a6f6-868b97c4e845","created":"2026-02-22T21:36:39.062947+0000","modified":"2026-02-22T21:36:46.090039+0000","last_up_change":"0.000000","last_in_change":"2026-02-22T21:36:39.629207+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"5ebb4bdc-b6e6-4276-a34c-a27c8810395c","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"24ae9b1a-c77d-483c-abb8-148e6becaaca","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"d5ca9f56-e467-4298-8a36-49cf123cc629","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-22T21:36:46.616 DEBUG:teuthology.misc:0 of 3 OSDs are up 2026-02-22T21:36:47.108 INFO:tasks.ceph.cluster2.osd.0.trial026.stderr:2026-02-22T21:36:47.107+0000 7ff4c1c26640 -1 osd.0 0 waiting for initial osdmap 2026-02-22T21:36:47.109 INFO:tasks.ceph.cluster2.osd.1.trial026.stderr:2026-02-22T21:36:47.107+0000 7f83088ef640 -1 osd.1 0 waiting for initial osdmap 2026-02-22T21:36:47.110 INFO:tasks.ceph.cluster2.osd.1.trial026.stderr:2026-02-22T21:36:47.109+0000 7f8300711640 -1 osd.1 8 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2026-02-22T21:36:47.110 INFO:tasks.ceph.cluster2.osd.0.trial026.stderr:2026-02-22T21:36:47.109+0000 7ff4b9a48640 -1 osd.0 8 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2026-02-22T21:36:52.920 DEBUG:teuthology.orchestra.run.trial026:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster2 osd dump --format=json 2026-02-22T21:36:53.071 INFO:teuthology.misc.health.trial026.stdout: 2026-02-22T21:36:53.072 INFO:teuthology.misc.health.trial026.stdout:{"epoch":12,"fsid":"2e52b784-3eb2-4856-a6f6-868b97c4e845","created":"2026-02-22T21:36:39.062947+0000","modified":"2026-02-22T21:36:51.101939+0000","last_up_change":"2026-02-22T21:36:48.098894+0000","last_in_change":"2026-02-22T21:36:39.629207+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-02-22T21:36:48.933035+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"5ebb4bdc-b6e6-4276-a34c-a27c8810395c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.26:6808","nonce":671216846},{"type":"v1","addr":"10.20.193.26:6809","nonce":671216846}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.26:6810","nonce":671216846},{"type":"v1","addr":"10.20.193.26:6811","nonce":671216846}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.26:6814","nonce":671216846},{"type":"v1","addr":"10.20.193.26:6815","nonce":671216846}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.26:6812","nonce":671216846},{"type":"v1","addr":"10.20.193.26:6813","nonce":671216846}]},"public_addr":"10.20.193.26:6809/671216846","cluster_addr":"10.20.193.26:6811/671216846","heartbeat_back_addr":"10.20.193.26:6815/671216846","heartbeat_front_addr":"10.20.193.26:6813/671216846","state":["exists","up"]},{"osd":1,"uuid":"24ae9b1a-c77d-483c-abb8-148e6becaaca","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":10,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.26:6816","nonce":4085277355},{"type":"v1","addr":"10.20.193.26:6817","nonce":4085277355}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.26:6818","nonce":4085277355},{"type":"v1","addr":"10.20.193.26:6819","nonce":4085277355}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.26:6822","nonce":4085277355},{"type":"v1","addr":"10.20.193.26:6823","nonce":4085277355}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.26:6820","nonce":4085277355},{"type":"v1","addr":"10.20.193.26:6821","nonce":4085277355}]},"public_addr":"10.20.193.26:6817/4085277355","cluster_addr":"10.20.193.26:6819/4085277355","heartbeat_back_addr":"10.20.193.26:6823/4085277355","heartbeat_front_addr":"10.20.193.26:6821/4085277355","state":["exists","up"]},{"osd":2,"uuid":"d5ca9f56-e467-4298-8a36-49cf123cc629","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.26:6800","nonce":4146597801},{"type":"v1","addr":"10.20.193.26:6801","nonce":4146597801}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.26:6802","nonce":4146597801},{"type":"v1","addr":"10.20.193.26:6803","nonce":4146597801}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.26:6806","nonce":4146597801},{"type":"v1","addr":"10.20.193.26:6807","nonce":4146597801}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.26:6804","nonce":4146597801},{"type":"v1","addr":"10.20.193.26:6805","nonce":4146597801}]},"public_addr":"10.20.193.26:6801/4146597801","cluster_addr":"10.20.193.26:6803/4146597801","heartbeat_back_addr":"10.20.193.26:6807/4146597801","heartbeat_front_addr":"10.20.193.26:6805/4146597801","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T21:36:46.235412+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T21:36:46.591328+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T21:36:46.061496+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-22T21:36:53.080 DEBUG:teuthology.misc:3 of 3 OSDs are up 2026-02-22T21:36:53.080 INFO:tasks.ceph:Creating RBD pool 2026-02-22T21:36:53.081 DEBUG:teuthology.orchestra.run.trial026:> sudo ceph --cluster cluster2 osd pool create rbd 8 2026-02-22T21:36:54.110 INFO:teuthology.orchestra.run.trial026.stderr:pool 'rbd' created 2026-02-22T21:36:54.122 DEBUG:teuthology.orchestra.run.trial026:> rbd --cluster cluster2 pool init rbd 2026-02-22T21:36:57.122 INFO:tasks.ceph:Starting mds daemons in cluster cluster2... 2026-02-22T21:36:57.123 INFO:tasks.ceph:Waiting until cluster2 daemons up and pgs clean... 2026-02-22T21:36:57.123 INFO:tasks.ceph.ceph_manager.cluster2:waiting for mgr available 2026-02-22T21:36:57.124 DEBUG:teuthology.orchestra.run.trial026:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 mgr dump --format=json 2026-02-22T21:36:57.124 INFO:tasks.daemonwatchdog.daemon_watchdog:watchdog starting 2026-02-22T21:36:57.304 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:36:57.312 INFO:teuthology.orchestra.run.trial026.stdout:{"epoch":5,"flags":0,"active_gid":4102,"active_name":"x","active_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.4:6826","nonce":3867516029},{"type":"v1","addr":"10.20.193.4:6827","nonce":3867516029}]},"active_addr":"10.20.193.4:6827/3867516029","active_change":"2026-02-22T21:36:40.912131+0000","active_mgr_features":4540701547738038271,"available":true,"standbys":[],"modules":["iostat","nfs","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"update_pg_upmap_activity":{"name":"update_pg_upmap_activity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Updates pg_upmap activity stats to be used in `balancer status detail`","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:10.4.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.7.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.2.5","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.51.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba":{"name":"container_image_samba","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-server:devbuilds-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba/SMB container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"quay.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"location of dashboards to include in grafana deployments","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"ssh_keepalive_count_max":{"name":"ssh_keepalive_count_max","type":"int","level":"advanced","flags":0,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"How many times ssh connections can fail liveness checks before the host is marked offline","long_desc":"","tags":[],"see_also":[]},"ssh_keepalive_interval":{"name":"ssh_keepalive_interval","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"How often ssh connections are checked for liveness","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":true,"error_string":"","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_requests":{"name":"max_requests","type":"int","level":"advanced","flags":0,"default_value":"500","min":"","max":"","enum_allowed":[],"desc":"Maximum number of requests to keep in memory. When new request comes in, the oldest request will be removed if the number of requests exceeds the max request number. if un-finished request is removed, error message will be logged in the ceph-mgr log.","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"secondary_zone_period_retry_limit":{"name":"secondary_zone_period_retry_limit","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"RGW module period update retry limit for secondary site","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"squid":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"force_disabled_modules":{},"last_failure_osd_epoch":0,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"10.20.193.4:0","nonce":3223018097}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"10.20.193.4:0","nonce":4239510854}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"10.20.193.4:0","nonce":3259022172}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"10.20.193.4:0","nonce":1753922590}]}]} 2026-02-22T21:36:57.319 INFO:tasks.ceph.ceph_manager.cluster2:mgr available! 2026-02-22T21:36:57.319 INFO:tasks.ceph.ceph_manager.cluster2:waiting for all up 2026-02-22T21:36:57.320 DEBUG:teuthology.orchestra.run.trial026:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd dump --format=json 2026-02-22T21:36:57.476 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:36:57.476 INFO:teuthology.orchestra.run.trial026.stdout:{"epoch":16,"fsid":"2e52b784-3eb2-4856-a6f6-868b97c4e845","created":"2026-02-22T21:36:39.062947+0000","modified":"2026-02-22T21:36:57.113364+0000","last_up_change":"2026-02-22T21:36:48.098894+0000","last_in_change":"2026-02-22T21:36:39.629207+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-02-22T21:36:48.933035+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2026-02-22T21:36:53.238866+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":16,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"5ebb4bdc-b6e6-4276-a34c-a27c8810395c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":13,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.26:6808","nonce":671216846},{"type":"v1","addr":"10.20.193.26:6809","nonce":671216846}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.26:6810","nonce":671216846},{"type":"v1","addr":"10.20.193.26:6811","nonce":671216846}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.26:6814","nonce":671216846},{"type":"v1","addr":"10.20.193.26:6815","nonce":671216846}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.26:6812","nonce":671216846},{"type":"v1","addr":"10.20.193.26:6813","nonce":671216846}]},"public_addr":"10.20.193.26:6809/671216846","cluster_addr":"10.20.193.26:6811/671216846","heartbeat_back_addr":"10.20.193.26:6815/671216846","heartbeat_front_addr":"10.20.193.26:6813/671216846","state":["exists","up"]},{"osd":1,"uuid":"24ae9b1a-c77d-483c-abb8-148e6becaaca","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":13,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.26:6816","nonce":4085277355},{"type":"v1","addr":"10.20.193.26:6817","nonce":4085277355}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.26:6818","nonce":4085277355},{"type":"v1","addr":"10.20.193.26:6819","nonce":4085277355}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.26:6822","nonce":4085277355},{"type":"v1","addr":"10.20.193.26:6823","nonce":4085277355}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.26:6820","nonce":4085277355},{"type":"v1","addr":"10.20.193.26:6821","nonce":4085277355}]},"public_addr":"10.20.193.26:6817/4085277355","cluster_addr":"10.20.193.26:6819/4085277355","heartbeat_back_addr":"10.20.193.26:6823/4085277355","heartbeat_front_addr":"10.20.193.26:6821/4085277355","state":["exists","up"]},{"osd":2,"uuid":"d5ca9f56-e467-4298-8a36-49cf123cc629","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":13,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.26:6800","nonce":4146597801},{"type":"v1","addr":"10.20.193.26:6801","nonce":4146597801}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.26:6802","nonce":4146597801},{"type":"v1","addr":"10.20.193.26:6803","nonce":4146597801}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.26:6806","nonce":4146597801},{"type":"v1","addr":"10.20.193.26:6807","nonce":4146597801}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.26:6804","nonce":4146597801},{"type":"v1","addr":"10.20.193.26:6805","nonce":4146597801}]},"public_addr":"10.20.193.26:6801/4146597801","cluster_addr":"10.20.193.26:6803/4146597801","heartbeat_back_addr":"10.20.193.26:6807/4146597801","heartbeat_front_addr":"10.20.193.26:6805/4146597801","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T21:36:46.235412+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T21:36:46.591328+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T21:36:46.061496+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-22T21:36:57.485 INFO:tasks.ceph.ceph_manager.cluster2:all up! 2026-02-22T21:36:57.485 DEBUG:teuthology.orchestra.run.trial026:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd dump --format=json 2026-02-22T21:36:57.642 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:36:57.642 INFO:teuthology.orchestra.run.trial026.stdout:{"epoch":16,"fsid":"2e52b784-3eb2-4856-a6f6-868b97c4e845","created":"2026-02-22T21:36:39.062947+0000","modified":"2026-02-22T21:36:57.113364+0000","last_up_change":"2026-02-22T21:36:48.098894+0000","last_in_change":"2026-02-22T21:36:39.629207+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-02-22T21:36:48.933035+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2026-02-22T21:36:53.238866+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":16,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"5ebb4bdc-b6e6-4276-a34c-a27c8810395c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":13,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.26:6808","nonce":671216846},{"type":"v1","addr":"10.20.193.26:6809","nonce":671216846}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.26:6810","nonce":671216846},{"type":"v1","addr":"10.20.193.26:6811","nonce":671216846}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.26:6814","nonce":671216846},{"type":"v1","addr":"10.20.193.26:6815","nonce":671216846}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.26:6812","nonce":671216846},{"type":"v1","addr":"10.20.193.26:6813","nonce":671216846}]},"public_addr":"10.20.193.26:6809/671216846","cluster_addr":"10.20.193.26:6811/671216846","heartbeat_back_addr":"10.20.193.26:6815/671216846","heartbeat_front_addr":"10.20.193.26:6813/671216846","state":["exists","up"]},{"osd":1,"uuid":"24ae9b1a-c77d-483c-abb8-148e6becaaca","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":13,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.26:6816","nonce":4085277355},{"type":"v1","addr":"10.20.193.26:6817","nonce":4085277355}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.26:6818","nonce":4085277355},{"type":"v1","addr":"10.20.193.26:6819","nonce":4085277355}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.26:6822","nonce":4085277355},{"type":"v1","addr":"10.20.193.26:6823","nonce":4085277355}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.26:6820","nonce":4085277355},{"type":"v1","addr":"10.20.193.26:6821","nonce":4085277355}]},"public_addr":"10.20.193.26:6817/4085277355","cluster_addr":"10.20.193.26:6819/4085277355","heartbeat_back_addr":"10.20.193.26:6823/4085277355","heartbeat_front_addr":"10.20.193.26:6821/4085277355","state":["exists","up"]},{"osd":2,"uuid":"d5ca9f56-e467-4298-8a36-49cf123cc629","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":13,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.26:6800","nonce":4146597801},{"type":"v1","addr":"10.20.193.26:6801","nonce":4146597801}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.26:6802","nonce":4146597801},{"type":"v1","addr":"10.20.193.26:6803","nonce":4146597801}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.26:6806","nonce":4146597801},{"type":"v1","addr":"10.20.193.26:6807","nonce":4146597801}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.26:6804","nonce":4146597801},{"type":"v1","addr":"10.20.193.26:6805","nonce":4146597801}]},"public_addr":"10.20.193.26:6801/4146597801","cluster_addr":"10.20.193.26:6803/4146597801","heartbeat_back_addr":"10.20.193.26:6807/4146597801","heartbeat_front_addr":"10.20.193.26:6805/4146597801","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T21:36:46.235412+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T21:36:46.591328+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T21:36:46.061496+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-22T21:36:57.651 DEBUG:teuthology.orchestra.run.trial026:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.0 flush_pg_stats 2026-02-22T21:36:57.651 DEBUG:teuthology.orchestra.run.trial026:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.1 flush_pg_stats 2026-02-22T21:36:57.652 DEBUG:teuthology.orchestra.run.trial026:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.2 flush_pg_stats 2026-02-22T21:36:57.739 INFO:teuthology.orchestra.run.trial026.stdout:38654705667 2026-02-22T21:36:57.740 DEBUG:teuthology.orchestra.run.trial026:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.0 2026-02-22T21:36:57.741 INFO:teuthology.orchestra.run.trial026.stdout:38654705668 2026-02-22T21:36:57.741 DEBUG:teuthology.orchestra.run.trial026:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.1 2026-02-22T21:36:57.742 INFO:teuthology.orchestra.run.trial026.stdout:34359738373 2026-02-22T21:36:57.743 DEBUG:teuthology.orchestra.run.trial026:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.2 2026-02-22T21:36:57.941 INFO:teuthology.orchestra.run.trial026.stdout:38654705667 2026-02-22T21:36:57.945 INFO:teuthology.orchestra.run.trial026.stdout:38654705666 2026-02-22T21:36:57.948 INFO:teuthology.orchestra.run.trial026.stdout:34359738371 2026-02-22T21:36:57.950 INFO:tasks.ceph.ceph_manager.cluster2:need seq 38654705668 got 38654705667 for osd.1 2026-02-22T21:36:57.954 INFO:tasks.ceph.ceph_manager.cluster2:need seq 38654705667 got 38654705666 for osd.0 2026-02-22T21:36:57.958 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738373 got 34359738371 for osd.2 2026-02-22T21:36:58.951 DEBUG:teuthology.orchestra.run.trial026:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.1 2026-02-22T21:36:58.955 DEBUG:teuthology.orchestra.run.trial026:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.0 2026-02-22T21:36:58.959 DEBUG:teuthology.orchestra.run.trial026:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.2 2026-02-22T21:36:59.114 INFO:teuthology.orchestra.run.trial026.stdout:38654705667 2026-02-22T21:36:59.124 INFO:tasks.ceph.ceph_manager.cluster2:need seq 38654705668 got 38654705667 for osd.1 2026-02-22T21:36:59.158 INFO:teuthology.orchestra.run.trial026.stdout:38654705667 2026-02-22T21:36:59.159 INFO:teuthology.orchestra.run.trial026.stdout:34359738373 2026-02-22T21:36:59.167 INFO:tasks.ceph.ceph_manager.cluster2:need seq 38654705667 got 38654705667 for osd.0 2026-02-22T21:36:59.168 DEBUG:teuthology.parallel:result is None 2026-02-22T21:36:59.169 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738373 got 34359738373 for osd.2 2026-02-22T21:36:59.169 DEBUG:teuthology.parallel:result is None 2026-02-22T21:37:00.125 DEBUG:teuthology.orchestra.run.trial026:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.1 2026-02-22T21:37:00.289 INFO:teuthology.orchestra.run.trial026.stdout:38654705668 2026-02-22T21:37:00.298 INFO:tasks.ceph.ceph_manager.cluster2:need seq 38654705668 got 38654705668 for osd.1 2026-02-22T21:37:00.298 DEBUG:teuthology.parallel:result is None 2026-02-22T21:37:00.298 INFO:tasks.ceph.ceph_manager.cluster2:waiting for clean 2026-02-22T21:37:00.298 DEBUG:teuthology.orchestra.run.trial026:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2026-02-22T21:37:00.454 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:37:00.454 INFO:teuthology.orchestra.run.trial026.stderr:dumped all 2026-02-22T21:37:00.462 INFO:teuthology.orchestra.run.trial026.stdout:{"pg_ready":true,"pg_map":{"version":20,"stamp":"2026-02-22T21:36:58.918450+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":81428,"kb_used_data":528,"kb_used_omap":4,"kb_used_meta":80827,"kb_avail":283034092,"statfs":{"total":289910292480,"available":289826910208,"internally_reserved":0,"allocated":540672,"data_stored":1302919,"data_compressed":19632,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":4782,"internal_metadata":82767186},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.809030"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T21:36:57.117998+0000","last_change":"2026-02-22T21:36:57.117998+0000","last_active":"2026-02-22T21:36:57.117998+0000","last_peered":"2026-02-22T21:36:57.117998+0000","last_clean":"2026-02-22T21:36:57.117998+0000","last_became_active":"2026-02-22T21:36:55.116298+0000","last_became_peered":"2026-02-22T21:36:55.116298+0000","last_unstale":"2026-02-22T21:36:57.117998+0000","last_undegraded":"2026-02-22T21:36:57.117998+0000","last_fullsized":"2026-02-22T21:36:57.117998+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:54.106037+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:54.106037+0000","last_clean_scrub_stamp":"2026-02-22T21:36:54.106037+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T02:34:53.292431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00037892999999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.6","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T21:36:57.118108+0000","last_change":"2026-02-22T21:36:57.118108+0000","last_active":"2026-02-22T21:36:57.118108+0000","last_peered":"2026-02-22T21:36:57.118108+0000","last_clean":"2026-02-22T21:36:57.118108+0000","last_became_active":"2026-02-22T21:36:55.116070+0000","last_became_peered":"2026-02-22T21:36:55.116070+0000","last_unstale":"2026-02-22T21:36:57.118108+0000","last_undegraded":"2026-02-22T21:36:57.118108+0000","last_fullsized":"2026-02-22T21:36:57.118108+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:54.106037+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:54.106037+0000","last_clean_scrub_stamp":"2026-02-22T21:36:54.106037+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T07:44:09.720355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048051700000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.5","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T21:36:57.118060+0000","last_change":"2026-02-22T21:36:57.118060+0000","last_active":"2026-02-22T21:36:57.118060+0000","last_peered":"2026-02-22T21:36:57.118060+0000","last_clean":"2026-02-22T21:36:57.118060+0000","last_became_active":"2026-02-22T21:36:55.116352+0000","last_became_peered":"2026-02-22T21:36:55.116352+0000","last_unstale":"2026-02-22T21:36:57.118060+0000","last_undegraded":"2026-02-22T21:36:57.118060+0000","last_fullsized":"2026-02-22T21:36:57.118060+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:54.106037+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:54.106037+0000","last_clean_scrub_stamp":"2026-02-22T21:36:54.106037+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T00:50:23.097350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00044389899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.4","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T21:36:57.117889+0000","last_change":"2026-02-22T21:36:57.117889+0000","last_active":"2026-02-22T21:36:57.117889+0000","last_peered":"2026-02-22T21:36:57.117889+0000","last_clean":"2026-02-22T21:36:57.117889+0000","last_became_active":"2026-02-22T21:36:55.116352+0000","last_became_peered":"2026-02-22T21:36:55.116352+0000","last_unstale":"2026-02-22T21:36:57.117889+0000","last_undegraded":"2026-02-22T21:36:57.117889+0000","last_fullsized":"2026-02-22T21:36:57.117889+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:54.106037+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:54.106037+0000","last_clean_scrub_stamp":"2026-02-22T21:36:54.106037+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T06:10:01.562314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029249000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.2","version":"16'2","reported_seq":21,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T21:36:57.117209+0000","last_change":"2026-02-22T21:36:57.116752+0000","last_active":"2026-02-22T21:36:57.117209+0000","last_peered":"2026-02-22T21:36:57.117209+0000","last_clean":"2026-02-22T21:36:57.117209+0000","last_became_active":"2026-02-22T21:36:55.116275+0000","last_became_peered":"2026-02-22T21:36:55.116275+0000","last_unstale":"2026-02-22T21:36:57.117209+0000","last_undegraded":"2026-02-22T21:36:57.117209+0000","last_fullsized":"2026-02-22T21:36:57.117209+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:54.106037+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:54.106037+0000","last_clean_scrub_stamp":"2026-02-22T21:36:54.106037+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T07:41:58.739291+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000424664,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.1","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T21:36:57.115375+0000","last_change":"2026-02-22T21:36:57.115505+0000","last_active":"2026-02-22T21:36:57.115375+0000","last_peered":"2026-02-22T21:36:57.115375+0000","last_clean":"2026-02-22T21:36:57.115375+0000","last_became_active":"2026-02-22T21:36:55.116210+0000","last_became_peered":"2026-02-22T21:36:55.116210+0000","last_unstale":"2026-02-22T21:36:57.115375+0000","last_undegraded":"2026-02-22T21:36:57.115375+0000","last_fullsized":"2026-02-22T21:36:57.115375+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:54.106037+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:54.106037+0000","last_clean_scrub_stamp":"2026-02-22T21:36:54.106037+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T02:21:43.966839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024309900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.0","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T21:36:57.115352+0000","last_change":"2026-02-22T21:36:57.115504+0000","last_active":"2026-02-22T21:36:57.115352+0000","last_peered":"2026-02-22T21:36:57.115352+0000","last_clean":"2026-02-22T21:36:57.115352+0000","last_became_active":"2026-02-22T21:36:55.116216+0000","last_became_peered":"2026-02-22T21:36:55.116216+0000","last_unstale":"2026-02-22T21:36:57.115352+0000","last_undegraded":"2026-02-22T21:36:57.115352+0000","last_fullsized":"2026-02-22T21:36:57.115352+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:54.106037+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:54.106037+0000","last_clean_scrub_stamp":"2026-02-22T21:36:54.106037+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T23:54:52.644119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000245073,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.3","version":"14'1","reported_seq":20,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T21:36:57.117874+0000","last_change":"2026-02-22T21:36:57.117874+0000","last_active":"2026-02-22T21:36:57.117874+0000","last_peered":"2026-02-22T21:36:57.117874+0000","last_clean":"2026-02-22T21:36:57.117874+0000","last_became_active":"2026-02-22T21:36:55.116006+0000","last_became_peered":"2026-02-22T21:36:55.116006+0000","last_unstale":"2026-02-22T21:36:57.117874+0000","last_undegraded":"2026-02-22T21:36:57.117874+0000","last_fullsized":"2026-02-22T21:36:57.117874+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:54.106037+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:54.106037+0000","last_clean_scrub_stamp":"2026-02-22T21:36:54.106037+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T03:58:14.400632+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029477300000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"1.0","version":"11'72","reported_seq":110,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T21:36:57.117507+0000","last_change":"2026-02-22T21:36:50.308369+0000","last_active":"2026-02-22T21:36:57.117507+0000","last_peered":"2026-02-22T21:36:57.117507+0000","last_clean":"2026-02-22T21:36:57.117507+0000","last_became_active":"2026-02-22T21:36:50.308099+0000","last_became_peered":"2026-02-22T21:36:50.308099+0000","last_unstale":"2026-02-22T21:36:57.117507+0000","last_undegraded":"2026-02-22T21:36:57.117507+0000","last_fullsized":"2026-02-22T21:36:57.117507+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:49.099350+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:49.099350+0000","last_clean_scrub_stamp":"2026-02-22T21:36:49.099350+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T09:35:05.112768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":1180736,"data_compressed":19632,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":1,"up_from":9,"seq":38654705668,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27160,"kb_used_data":196,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94344680,"statfs":{"total":96636764160,"available":96608952320,"internally_reserved":0,"allocated":200704,"data_stored":631102,"data_compressed":9816,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1594,"internal_metadata":27589062},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":9,"seq":38654705667,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27168,"kb_used_data":196,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94344672,"statfs":{"total":96636764160,"available":96608944128,"internally_reserved":0,"allocated":200704,"data_stored":631102,"data_compressed":9816,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1594,"internal_metadata":27589062},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":8,"seq":34359738373,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27100,"kb_used_data":136,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94344740,"statfs":{"total":96636764160,"available":96609013760,"internally_reserved":0,"allocated":139264,"data_stored":40715,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27589062},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":9816,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":9816,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-22T21:37:00.464 DEBUG:teuthology.orchestra.run.trial026:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2026-02-22T21:37:00.619 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:37:00.619 INFO:teuthology.orchestra.run.trial026.stderr:dumped all 2026-02-22T21:37:00.626 INFO:teuthology.orchestra.run.trial026.stdout:{"pg_ready":true,"pg_map":{"version":20,"stamp":"2026-02-22T21:36:58.918450+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":81428,"kb_used_data":528,"kb_used_omap":4,"kb_used_meta":80827,"kb_avail":283034092,"statfs":{"total":289910292480,"available":289826910208,"internally_reserved":0,"allocated":540672,"data_stored":1302919,"data_compressed":19632,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":4782,"internal_metadata":82767186},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.809030"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T21:36:57.117998+0000","last_change":"2026-02-22T21:36:57.117998+0000","last_active":"2026-02-22T21:36:57.117998+0000","last_peered":"2026-02-22T21:36:57.117998+0000","last_clean":"2026-02-22T21:36:57.117998+0000","last_became_active":"2026-02-22T21:36:55.116298+0000","last_became_peered":"2026-02-22T21:36:55.116298+0000","last_unstale":"2026-02-22T21:36:57.117998+0000","last_undegraded":"2026-02-22T21:36:57.117998+0000","last_fullsized":"2026-02-22T21:36:57.117998+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:54.106037+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:54.106037+0000","last_clean_scrub_stamp":"2026-02-22T21:36:54.106037+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T02:34:53.292431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00037892999999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.6","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T21:36:57.118108+0000","last_change":"2026-02-22T21:36:57.118108+0000","last_active":"2026-02-22T21:36:57.118108+0000","last_peered":"2026-02-22T21:36:57.118108+0000","last_clean":"2026-02-22T21:36:57.118108+0000","last_became_active":"2026-02-22T21:36:55.116070+0000","last_became_peered":"2026-02-22T21:36:55.116070+0000","last_unstale":"2026-02-22T21:36:57.118108+0000","last_undegraded":"2026-02-22T21:36:57.118108+0000","last_fullsized":"2026-02-22T21:36:57.118108+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:54.106037+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:54.106037+0000","last_clean_scrub_stamp":"2026-02-22T21:36:54.106037+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T07:44:09.720355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048051700000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.5","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T21:36:57.118060+0000","last_change":"2026-02-22T21:36:57.118060+0000","last_active":"2026-02-22T21:36:57.118060+0000","last_peered":"2026-02-22T21:36:57.118060+0000","last_clean":"2026-02-22T21:36:57.118060+0000","last_became_active":"2026-02-22T21:36:55.116352+0000","last_became_peered":"2026-02-22T21:36:55.116352+0000","last_unstale":"2026-02-22T21:36:57.118060+0000","last_undegraded":"2026-02-22T21:36:57.118060+0000","last_fullsized":"2026-02-22T21:36:57.118060+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:54.106037+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:54.106037+0000","last_clean_scrub_stamp":"2026-02-22T21:36:54.106037+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T00:50:23.097350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00044389899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.4","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T21:36:57.117889+0000","last_change":"2026-02-22T21:36:57.117889+0000","last_active":"2026-02-22T21:36:57.117889+0000","last_peered":"2026-02-22T21:36:57.117889+0000","last_clean":"2026-02-22T21:36:57.117889+0000","last_became_active":"2026-02-22T21:36:55.116352+0000","last_became_peered":"2026-02-22T21:36:55.116352+0000","last_unstale":"2026-02-22T21:36:57.117889+0000","last_undegraded":"2026-02-22T21:36:57.117889+0000","last_fullsized":"2026-02-22T21:36:57.117889+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:54.106037+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:54.106037+0000","last_clean_scrub_stamp":"2026-02-22T21:36:54.106037+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T06:10:01.562314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029249000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.2","version":"16'2","reported_seq":21,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T21:36:57.117209+0000","last_change":"2026-02-22T21:36:57.116752+0000","last_active":"2026-02-22T21:36:57.117209+0000","last_peered":"2026-02-22T21:36:57.117209+0000","last_clean":"2026-02-22T21:36:57.117209+0000","last_became_active":"2026-02-22T21:36:55.116275+0000","last_became_peered":"2026-02-22T21:36:55.116275+0000","last_unstale":"2026-02-22T21:36:57.117209+0000","last_undegraded":"2026-02-22T21:36:57.117209+0000","last_fullsized":"2026-02-22T21:36:57.117209+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:54.106037+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:54.106037+0000","last_clean_scrub_stamp":"2026-02-22T21:36:54.106037+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T07:41:58.739291+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000424664,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.1","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T21:36:57.115375+0000","last_change":"2026-02-22T21:36:57.115505+0000","last_active":"2026-02-22T21:36:57.115375+0000","last_peered":"2026-02-22T21:36:57.115375+0000","last_clean":"2026-02-22T21:36:57.115375+0000","last_became_active":"2026-02-22T21:36:55.116210+0000","last_became_peered":"2026-02-22T21:36:55.116210+0000","last_unstale":"2026-02-22T21:36:57.115375+0000","last_undegraded":"2026-02-22T21:36:57.115375+0000","last_fullsized":"2026-02-22T21:36:57.115375+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:54.106037+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:54.106037+0000","last_clean_scrub_stamp":"2026-02-22T21:36:54.106037+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T02:21:43.966839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024309900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.0","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T21:36:57.115352+0000","last_change":"2026-02-22T21:36:57.115504+0000","last_active":"2026-02-22T21:36:57.115352+0000","last_peered":"2026-02-22T21:36:57.115352+0000","last_clean":"2026-02-22T21:36:57.115352+0000","last_became_active":"2026-02-22T21:36:55.116216+0000","last_became_peered":"2026-02-22T21:36:55.116216+0000","last_unstale":"2026-02-22T21:36:57.115352+0000","last_undegraded":"2026-02-22T21:36:57.115352+0000","last_fullsized":"2026-02-22T21:36:57.115352+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:54.106037+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:54.106037+0000","last_clean_scrub_stamp":"2026-02-22T21:36:54.106037+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T23:54:52.644119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000245073,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.3","version":"14'1","reported_seq":20,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T21:36:57.117874+0000","last_change":"2026-02-22T21:36:57.117874+0000","last_active":"2026-02-22T21:36:57.117874+0000","last_peered":"2026-02-22T21:36:57.117874+0000","last_clean":"2026-02-22T21:36:57.117874+0000","last_became_active":"2026-02-22T21:36:55.116006+0000","last_became_peered":"2026-02-22T21:36:55.116006+0000","last_unstale":"2026-02-22T21:36:57.117874+0000","last_undegraded":"2026-02-22T21:36:57.117874+0000","last_fullsized":"2026-02-22T21:36:57.117874+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:54.106037+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:54.106037+0000","last_clean_scrub_stamp":"2026-02-22T21:36:54.106037+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T03:58:14.400632+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029477300000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"1.0","version":"11'72","reported_seq":110,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T21:36:57.117507+0000","last_change":"2026-02-22T21:36:50.308369+0000","last_active":"2026-02-22T21:36:57.117507+0000","last_peered":"2026-02-22T21:36:57.117507+0000","last_clean":"2026-02-22T21:36:57.117507+0000","last_became_active":"2026-02-22T21:36:50.308099+0000","last_became_peered":"2026-02-22T21:36:50.308099+0000","last_unstale":"2026-02-22T21:36:57.117507+0000","last_undegraded":"2026-02-22T21:36:57.117507+0000","last_fullsized":"2026-02-22T21:36:57.117507+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:49.099350+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:49.099350+0000","last_clean_scrub_stamp":"2026-02-22T21:36:49.099350+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T09:35:05.112768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":1180736,"data_compressed":19632,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":1,"up_from":9,"seq":38654705668,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27160,"kb_used_data":196,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94344680,"statfs":{"total":96636764160,"available":96608952320,"internally_reserved":0,"allocated":200704,"data_stored":631102,"data_compressed":9816,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1594,"internal_metadata":27589062},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":9,"seq":38654705667,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27168,"kb_used_data":196,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94344672,"statfs":{"total":96636764160,"available":96608944128,"internally_reserved":0,"allocated":200704,"data_stored":631102,"data_compressed":9816,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1594,"internal_metadata":27589062},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":8,"seq":34359738373,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27100,"kb_used_data":136,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94344740,"statfs":{"total":96636764160,"available":96609013760,"internally_reserved":0,"allocated":139264,"data_stored":40715,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27589062},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":9816,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":9816,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-22T21:37:00.627 INFO:tasks.ceph.ceph_manager.cluster2:clean! 2026-02-22T21:37:00.627 INFO:tasks.ceph:Waiting until ceph cluster cluster2 is healthy... 2026-02-22T21:37:00.627 INFO:tasks.ceph.ceph_manager.cluster2:wait_until_healthy 2026-02-22T21:37:00.627 DEBUG:teuthology.orchestra.run.trial026:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 health --format=json 2026-02-22T21:37:00.799 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T21:37:00.799 INFO:teuthology.orchestra.run.trial026.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2026-02-22T21:37:00.807 INFO:tasks.ceph.ceph_manager.cluster2:wait_until_healthy done 2026-02-22T21:37:00.807 INFO:teuthology.run_tasks:Running task exec... 2026-02-22T21:37:00.823 INFO:teuthology.task.exec:Executing custom commands... 2026-02-22T21:37:00.823 INFO:teuthology.task.exec:Running commands on role cluster1.client.mirror.0 host ubuntu@trial026.front.sepia.ceph.com 2026-02-22T21:37:00.824 DEBUG:teuthology.orchestra.run.trial026:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror mon '"'"'profile rbd-mirror-peer'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T21:37:01.016 INFO:teuthology.orchestra.run.trial026.stdout:[client.mirror] 2026-02-22T21:37:01.016 INFO:teuthology.orchestra.run.trial026.stdout: key = AQCzdptpKJe9KhAApC05qr2GvyvojF3F7P0D2Q== 2026-02-22T21:37:01.017 INFO:teuthology.orchestra.run.trial026.stdout: caps mon = "profile rbd-mirror-peer" 2026-02-22T21:37:01.017 INFO:teuthology.orchestra.run.trial026.stdout: caps osd = "profile rbd" 2026-02-22T21:37:01.017 INFO:teuthology.orchestra.run.trial026.stderr:updated caps for client.mirror 2026-02-22T21:37:01.025 DEBUG:teuthology.orchestra.run.trial026:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.0 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T21:37:01.226 INFO:teuthology.orchestra.run.trial026.stdout:[client.mirror.0] 2026-02-22T21:37:01.226 INFO:teuthology.orchestra.run.trial026.stdout: key = AQCzdptpipfiLxAAjzB/u0QwZJmrhV96Y++AEg== 2026-02-22T21:37:01.226 INFO:teuthology.orchestra.run.trial026.stdout: caps mon = "profile rbd-mirror" 2026-02-22T21:37:01.226 INFO:teuthology.orchestra.run.trial026.stdout: caps osd = "profile rbd" 2026-02-22T21:37:01.226 INFO:teuthology.orchestra.run.trial026.stderr:updated caps for client.mirror.0 2026-02-22T21:37:01.235 DEBUG:teuthology.orchestra.run.trial026:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.1 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T21:37:01.429 INFO:teuthology.orchestra.run.trial026.stdout:[client.mirror.1] 2026-02-22T21:37:01.430 INFO:teuthology.orchestra.run.trial026.stdout: key = AQCzdptpflQgNRAAtzGrTX/PFYfDsxh6+zboAw== 2026-02-22T21:37:01.430 INFO:teuthology.orchestra.run.trial026.stdout: caps mon = "profile rbd-mirror" 2026-02-22T21:37:01.430 INFO:teuthology.orchestra.run.trial026.stdout: caps osd = "profile rbd" 2026-02-22T21:37:01.430 INFO:teuthology.orchestra.run.trial026.stderr:updated caps for client.mirror.1 2026-02-22T21:37:01.439 DEBUG:teuthology.orchestra.run.trial026:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.2 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T21:37:01.631 INFO:teuthology.orchestra.run.trial026.stdout:[client.mirror.2] 2026-02-22T21:37:01.631 INFO:teuthology.orchestra.run.trial026.stdout: key = AQCzdptpibgiOhAABk6NsI44ocuIgNnj39mS+Q== 2026-02-22T21:37:01.631 INFO:teuthology.orchestra.run.trial026.stdout: caps mon = "profile rbd-mirror" 2026-02-22T21:37:01.631 INFO:teuthology.orchestra.run.trial026.stdout: caps osd = "profile rbd" 2026-02-22T21:37:01.631 INFO:teuthology.orchestra.run.trial026.stderr:updated caps for client.mirror.2 2026-02-22T21:37:01.639 DEBUG:teuthology.orchestra.run.trial026:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.3 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T21:37:01.835 INFO:teuthology.orchestra.run.trial026.stdout:[client.mirror.3] 2026-02-22T21:37:01.835 INFO:teuthology.orchestra.run.trial026.stdout: key = AQC0dptpPNy4AxAATEmDA2RB4Tnwt9QgC5zUUA== 2026-02-22T21:37:01.836 INFO:teuthology.orchestra.run.trial026.stdout: caps mon = "profile rbd-mirror" 2026-02-22T21:37:01.836 INFO:teuthology.orchestra.run.trial026.stdout: caps osd = "profile rbd" 2026-02-22T21:37:01.836 INFO:teuthology.orchestra.run.trial026.stderr:updated caps for client.mirror.3 2026-02-22T21:37:01.845 DEBUG:teuthology.orchestra.run.trial026:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror mon '"'"'profile rbd-mirror-peer'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T21:37:02.037 INFO:teuthology.orchestra.run.trial026.stdout:[client.mirror] 2026-02-22T21:37:02.037 INFO:teuthology.orchestra.run.trial026.stdout: key = AQDYdptpgzY6CxAAQJtwbthKMKzokmnKMHmSnQ== 2026-02-22T21:37:02.037 INFO:teuthology.orchestra.run.trial026.stdout: caps mon = "profile rbd-mirror-peer" 2026-02-22T21:37:02.038 INFO:teuthology.orchestra.run.trial026.stdout: caps osd = "profile rbd" 2026-02-22T21:37:02.038 INFO:teuthology.orchestra.run.trial026.stderr:updated caps for client.mirror 2026-02-22T21:37:02.047 DEBUG:teuthology.orchestra.run.trial026:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.0 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T21:37:02.241 INFO:teuthology.orchestra.run.trial026.stdout:[client.mirror.0] 2026-02-22T21:37:02.241 INFO:teuthology.orchestra.run.trial026.stdout: key = AQDYdptpSjloEBAAzbY3aIIsPqDMUlGlyoJU3w== 2026-02-22T21:37:02.241 INFO:teuthology.orchestra.run.trial026.stdout: caps mon = "profile rbd-mirror" 2026-02-22T21:37:02.241 INFO:teuthology.orchestra.run.trial026.stdout: caps osd = "profile rbd" 2026-02-22T21:37:02.241 INFO:teuthology.orchestra.run.trial026.stderr:updated caps for client.mirror.0 2026-02-22T21:37:02.250 DEBUG:teuthology.orchestra.run.trial026:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.1 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T21:37:02.450 INFO:teuthology.orchestra.run.trial026.stdout:[client.mirror.1] 2026-02-22T21:37:02.450 INFO:teuthology.orchestra.run.trial026.stdout: key = AQDYdptpCRdzFRAAsLYLN5bi/C8V+ae6X66s9w== 2026-02-22T21:37:02.450 INFO:teuthology.orchestra.run.trial026.stdout: caps mon = "profile rbd-mirror" 2026-02-22T21:37:02.450 INFO:teuthology.orchestra.run.trial026.stdout: caps osd = "profile rbd" 2026-02-22T21:37:02.450 INFO:teuthology.orchestra.run.trial026.stderr:updated caps for client.mirror.1 2026-02-22T21:37:02.459 DEBUG:teuthology.orchestra.run.trial026:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.2 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T21:37:02.652 INFO:teuthology.orchestra.run.trial026.stdout:[client.mirror.2] 2026-02-22T21:37:02.653 INFO:teuthology.orchestra.run.trial026.stdout: key = AQDYdptpSIqlGhAAh1xP+oEdKaG+ueEzQh82BA== 2026-02-22T21:37:02.653 INFO:teuthology.orchestra.run.trial026.stdout: caps mon = "profile rbd-mirror" 2026-02-22T21:37:02.653 INFO:teuthology.orchestra.run.trial026.stdout: caps osd = "profile rbd" 2026-02-22T21:37:02.653 INFO:teuthology.orchestra.run.trial026.stderr:updated caps for client.mirror.2 2026-02-22T21:37:02.662 DEBUG:teuthology.orchestra.run.trial026:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.3 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T21:37:02.856 INFO:teuthology.orchestra.run.trial026.stdout:[client.mirror.3] 2026-02-22T21:37:02.856 INFO:teuthology.orchestra.run.trial026.stdout: key = AQDYdptpJtKlHxAAqCaG2hbinah7q3snnYlvhQ== 2026-02-22T21:37:02.856 INFO:teuthology.orchestra.run.trial026.stdout: caps mon = "profile rbd-mirror" 2026-02-22T21:37:02.856 INFO:teuthology.orchestra.run.trial026.stdout: caps osd = "profile rbd" 2026-02-22T21:37:02.857 INFO:teuthology.orchestra.run.trial026.stderr:updated caps for client.mirror.3 2026-02-22T21:37:02.866 INFO:teuthology.run_tasks:Running task rbd-mirror... 2026-02-22T21:37:02.884 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Restarting daemon 2026-02-22T21:37:02.884 DEBUG:teuthology.orchestra.run.trial026:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 2026-02-22T21:37:02.887 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Started 2026-02-22T21:37:02.887 INFO:teuthology.run_tasks:Running task rbd-mirror... 2026-02-22T21:37:02.902 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Restarting daemon 2026-02-22T21:37:02.902 DEBUG:teuthology.orchestra.run.trial026:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 2026-02-22T21:37:02.905 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Started 2026-02-22T21:37:02.905 INFO:teuthology.run_tasks:Running task rbd-mirror... 2026-02-22T21:37:02.920 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Restarting daemon 2026-02-22T21:37:02.920 DEBUG:teuthology.orchestra.run.trial026:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.2 2026-02-22T21:37:02.924 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Started 2026-02-22T21:37:02.924 INFO:teuthology.run_tasks:Running task rbd-mirror... 2026-02-22T21:37:02.940 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Restarting daemon 2026-02-22T21:37:02.940 DEBUG:teuthology.orchestra.run.trial026:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.3 2026-02-22T21:37:02.943 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Started 2026-02-22T21:37:02.943 INFO:teuthology.run_tasks:Running task rbd-mirror... 2026-02-22T21:37:02.959 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Restarting daemon 2026-02-22T21:37:02.959 DEBUG:teuthology.orchestra.run.trial026:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 2026-02-22T21:37:02.962 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Started 2026-02-22T21:37:02.962 INFO:teuthology.run_tasks:Running task rbd-mirror... 2026-02-22T21:37:03.026 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Restarting daemon 2026-02-22T21:37:03.026 DEBUG:teuthology.orchestra.run.trial026:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 2026-02-22T21:37:03.028 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Started 2026-02-22T21:37:03.028 INFO:teuthology.run_tasks:Running task rbd-mirror... 2026-02-22T21:37:03.040 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Restarting daemon 2026-02-22T21:37:03.040 DEBUG:teuthology.orchestra.run.trial026:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2 2026-02-22T21:37:03.041 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Started 2026-02-22T21:37:03.041 INFO:teuthology.run_tasks:Running task rbd-mirror... 2026-02-22T21:37:03.053 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Restarting daemon 2026-02-22T21:37:03.053 DEBUG:teuthology.orchestra.run.trial026:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.3 2026-02-22T21:37:03.055 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Started 2026-02-22T21:37:03.055 INFO:teuthology.run_tasks:Running task rbd-mirror-thrash... 2026-02-22T21:37:03.070 INFO:tasks.rbd_mirror_thrash:rbd_mirror_thrash using random seed: 1771796223 2026-02-22T21:37:03.070 DEBUG:tasks.rbd_mirror_thrash:Yielding 2026-02-22T21:37:03.070 INFO:teuthology.run_tasks:Running task rbd-mirror-thrash... 2026-02-22T21:37:03.083 INFO:tasks.rbd_mirror_thrash:rbd_mirror_thrash using random seed: 1771796223 2026-02-22T21:37:03.083 DEBUG:tasks.rbd_mirror_thrash:Yielding 2026-02-22T21:37:03.083 INFO:teuthology.run_tasks:Running task workunit... 2026-02-22T21:37:03.104 INFO:tasks.workunit:Pulling workunits from ref f90f4eaebb28b619cf76e80cbbe997656da69579 2026-02-22T21:37:03.104 INFO:tasks.workunit:Making a separate scratch dir for every client... 2026-02-22T21:37:03.105 DEBUG:teuthology.orchestra.run.trial026:> stat -- /home/ubuntu/cephtest/mnt.cluster1.mirror 2026-02-22T21:37:03.105 INFO:tasks.rbd_mirror_thrash:starting thrash for cluster cluster1 2026-02-22T21:37:03.105 INFO:tasks.rbd_mirror_thrash:waiting for 96.22108395828417 secs before thrashing 2026-02-22T21:37:03.105 INFO:tasks.rbd_mirror_thrash:starting thrash for cluster cluster2 2026-02-22T21:37:03.105 INFO:tasks.rbd_mirror_thrash:waiting for 111.21611295667971 secs before thrashing 2026-02-22T21:37:03.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-22T21:37:03.121 INFO:teuthology.orchestra.run.trial026.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.cluster1.mirror': No such file or directory 2026-02-22T21:37:03.121 DEBUG:teuthology.orchestra.run.trial026:> mkdir -- /home/ubuntu/cephtest/mnt.cluster1.mirror 2026-02-22T21:37:03.178 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.cluster1.mirror 2026-02-22T21:37:03.178 DEBUG:teuthology.orchestra.run.trial026:> cd -- /home/ubuntu/cephtest/mnt.cluster1.mirror && mkdir -- client.mirror 2026-02-22T21:37:03.234 INFO:tasks.workunit:timeout=6h 2026-02-22T21:37:03.235 INFO:tasks.workunit:cleanup=True 2026-02-22T21:37:03.235 DEBUG:teuthology.orchestra.run.trial026:> rm -rf /home/ubuntu/cephtest/clone.cluster1.client.mirror && git clone https://github.com/ceph/ceph-ci.git /home/ubuntu/cephtest/clone.cluster1.client.mirror && cd /home/ubuntu/cephtest/clone.cluster1.client.mirror && git checkout f90f4eaebb28b619cf76e80cbbe997656da69579 2026-02-22T21:37:03.297 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Cloning into '/home/ubuntu/cephtest/clone.cluster1.client.mirror'... 2026-02-22T21:37:26.391 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Note: switching to 'f90f4eaebb28b619cf76e80cbbe997656da69579'. 2026-02-22T21:37:26.391 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T21:37:26.391 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:You are in 'detached HEAD' state. You can look around, make experimental 2026-02-22T21:37:26.391 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:changes and commit them, and you can discard any commits you make in this 2026-02-22T21:37:26.391 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:state without impacting any branches by switching back to a branch. 2026-02-22T21:37:26.391 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T21:37:26.391 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:If you want to create a new branch to retain commits you create, you may 2026-02-22T21:37:26.392 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:do so (now or later) by using -c with the switch command. Example: 2026-02-22T21:37:26.392 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T21:37:26.392 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: git switch -c 2026-02-22T21:37:26.392 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T21:37:26.392 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Or undo this operation with: 2026-02-22T21:37:26.392 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T21:37:26.392 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: git switch - 2026-02-22T21:37:26.392 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T21:37:26.392 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Turn off this advice by setting config variable advice.detachedHead to false 2026-02-22T21:37:26.392 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T21:37:26.392 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:HEAD is now at f90f4eaebb2 Merge branch 'wip-74712-squid' of https://github.com/idryomov/ceph into wip-yuri6-testing-2026-02-22-1629-squid 2026-02-22T21:37:26.398 DEBUG:teuthology.orchestra.run.trial026:> cd -- /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.cluster1.client.mirror 2026-02-22T21:37:26.455 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2026-02-22T21:37:26.457 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/direct_io' 2026-02-22T21:37:26.457 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2026-02-22T21:37:26.552 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2026-02-22T21:37:26.582 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2026-02-22T21:37:26.605 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/direct_io' 2026-02-22T21:37:26.606 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/fs' 2026-02-22T21:37:26.606 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2026-02-22T21:37:26.632 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/fs' 2026-02-22T21:37:26.636 DEBUG:teuthology.orchestra.run.trial026:> set -ex 2026-02-22T21:37:26.636 DEBUG:teuthology.orchestra.run.trial026:> dd if=/home/ubuntu/cephtest/workunits.list.cluster1.client.mirror of=/dev/stdout 2026-02-22T21:37:26.693 INFO:tasks.workunit:Running workunits matching rbd/rbd_mirror_stress.sh on cluster1.client.mirror... 2026-02-22T21:37:26.698 INFO:tasks.workunit:Running workunit rbd/rbd_mirror_stress.sh... 2026-02-22T21:37:26.698 DEBUG:teuthology.orchestra.run.trial026:workunit test rbd/rbd_mirror_stress.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror/tmp && cd -- /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=f90f4eaebb28b619cf76e80cbbe997656da69579 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster cluster1" CEPH_ID="mirror" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.cluster1.client.mirror CEPH_ROOT=/home/ubuntu/cephtest/clone.cluster1.client.mirror CEPH_MNT=/home/ubuntu/cephtest/mnt.cluster1.mirror CEPH_ARGS='' MIRROR_POOL_MODE=image RBD_IMAGE_FEATURES=layering,exclusive-lock RBD_MIRROR_INSTANCES=4 RBD_MIRROR_MODE=snapshot RBD_MIRROR_USE_EXISTING_CLUSTER=1 RBD_MIRROR_USE_RBD_MIRROR=1 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 6h /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/rbd_mirror_stress.sh 2026-02-22T21:37:26.753 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ IMAGE_COUNT=50 2026-02-22T21:37:26.753 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ export LOCKDEP=0 2026-02-22T21:37:26.753 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ LOCKDEP=0 2026-02-22T21:37:26.754 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ dirname /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/rbd_mirror_stress.sh 2026-02-22T21:37:26.754 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ . /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/rbd_mirror_helpers.sh 2026-02-22T21:37:26.755 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ type xmlstarlet 2026-02-22T21:37:26.755 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ XMLSTARLET=xmlstarlet 2026-02-22T21:37:26.755 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ RBD_MIRROR_INSTANCES=4 2026-02-22T21:37:26.755 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ CLUSTER1=cluster1 2026-02-22T21:37:26.755 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ CLUSTER2=cluster2 2026-02-22T21:37:26.755 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ PEER_CLUSTER_SUFFIX= 2026-02-22T21:37:26.755 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ POOL=mirror 2026-02-22T21:37:26.755 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ PARENT_POOL=mirror_parent 2026-02-22T21:37:26.755 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ NS1=ns1 2026-02-22T21:37:26.755 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ NS2=ns2 2026-02-22T21:37:26.755 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ TEMPDIR= 2026-02-22T21:37:26.755 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ CEPH_ID=mirror 2026-02-22T21:37:26.755 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ RBD_IMAGE_FEATURES=layering,exclusive-lock 2026-02-22T21:37:26.755 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ MIRROR_USER_ID_PREFIX=mirror. 2026-02-22T21:37:26.755 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ RBD_MIRROR_MODE=snapshot 2026-02-22T21:37:26.755 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ MIRROR_POOL_MODE=image 2026-02-22T21:37:26.755 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' snapshot = snapshot ']' 2026-02-22T21:37:26.755 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ MIRROR_POOL_MODE=image 2026-02-22T21:37:26.756 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ export 'CEPH_ARGS=--id mirror' 2026-02-22T21:37:26.756 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ CEPH_ARGS='--id mirror' 2026-02-22T21:37:26.756 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ LAST_MIRROR_INSTANCE=3 2026-02-22T21:37:26.756 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ dirname /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/rbd_mirror_stress.sh 2026-02-22T21:37:26.756 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ readlink -f /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/../../../src 2026-02-22T21:37:26.756 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ CEPH_ROOT=/home/ubuntu/cephtest/clone.cluster1.client.mirror/src 2026-02-22T21:37:26.756 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ CEPH_BIN=. 2026-02-22T21:37:26.756 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ CEPH_SRC=. 2026-02-22T21:37:26.756 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' -e CMakeCache.txt ']' 2026-02-22T21:37:26.758 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 0 -gt 0 ']' 2026-02-22T21:37:26.758 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ setup 2026-02-22T21:37:26.758 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local c 2026-02-22T21:37:26.758 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ trap 'cleanup $?' INT TERM EXIT 2026-02-22T21:37:26.758 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ setup_tempdir 2026-02-22T21:37:26.758 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n '' ']' 2026-02-22T21:37:26.758 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mktemp -d 2026-02-22T21:37:26.759 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ TEMPDIR=/tmp/tmp.j44LZPowal 2026-02-22T21:37:26.759 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -z 1 ']' 2026-02-22T21:37:26.759 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ setup_pools cluster1 cluster2 2026-02-22T21:37:26.759 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:37:26.759 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local remote_cluster=cluster2 2026-02-22T21:37:26.759 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mon_map_file 2026-02-22T21:37:26.759 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mon_addr 2026-02-22T21:37:26.759 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local admin_key_file 2026-02-22T21:37:26.759 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local uuid 2026-02-22T21:37:26.759 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ CEPH_ARGS= 2026-02-22T21:37:26.759 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ ceph --cluster cluster1 osd pool create mirror 64 64 2026-02-22T21:37:27.397 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:pool 'mirror' already exists 2026-02-22T21:37:27.405 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ CEPH_ARGS= 2026-02-22T21:37:27.405 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ ceph --cluster cluster1 osd pool create mirror_parent 64 64 2026-02-22T21:37:28.400 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:pool 'mirror_parent' already exists 2026-02-22T21:37:28.408 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ CEPH_ARGS= 2026-02-22T21:37:28.408 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 pool init mirror 2026-02-22T21:37:31.369 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ CEPH_ARGS= 2026-02-22T21:37:31.369 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 pool init mirror_parent 2026-02-22T21:37:34.374 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n '' ']' 2026-02-22T21:37:34.374 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ CEPH_ARGS= 2026-02-22T21:37:34.374 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 mirror pool enable --site-name cluster1 mirror image 2026-02-22T21:37:34.402 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 mirror pool enable mirror_parent image 2026-02-22T21:37:34.427 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 namespace create mirror/ns1 2026-02-22T21:37:34.452 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 namespace create mirror/ns2 2026-02-22T21:37:34.476 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 namespace create mirror_parent/ns1 2026-02-22T21:37:34.500 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 mirror pool enable mirror/ns1 image 2026-02-22T21:37:34.728 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 mirror pool enable mirror/ns2 image 2026-02-22T21:37:34.753 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 mirror pool enable mirror_parent/ns1 image 2026-02-22T21:37:34.779 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -z ']' 2026-02-22T21:37:34.780 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -z ']' 2026-02-22T21:37:34.780 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ peer_add cluster1 mirror cluster2 2026-02-22T21:37:34.780 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:37:34.780 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:37:34.780 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:37:34.780 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:37:34.780 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local client_cluster=cluster2 2026-02-22T21:37:34.780 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:37:34.780 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local remote_cluster=cluster2 2026-02-22T21:37:34.780 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local uuid_var_name 2026-02-22T21:37:34.780 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n '' ']' 2026-02-22T21:37:34.780 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local error_code 2026-02-22T21:37:34.780 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local peer_uuid 2026-02-22T21:37:34.780 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 16 32 2026-02-22T21:37:34.780 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set +e 2026-02-22T21:37:34.780 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster cluster1 mirror pool peer add mirror cluster2 2026-02-22T21:37:34.803 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ peer_uuid=34f2087e-7a83-4a78-877d-ab02a7cd2992 2026-02-22T21:37:34.803 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ error_code=0 2026-02-22T21:37:34.803 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set -e 2026-02-22T21:37:34.803 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' 0 -eq 17 ']' 2026-02-22T21:37:34.803 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 0 -eq 0 2026-02-22T21:37:34.803 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n '' ']' 2026-02-22T21:37:34.803 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:37:34.803 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ peer_add cluster1 mirror_parent cluster2 2026-02-22T21:37:34.803 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:37:34.803 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:37:34.803 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror_parent 2026-02-22T21:37:34.803 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:37:34.803 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local client_cluster=cluster2 2026-02-22T21:37:34.803 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:37:34.804 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local remote_cluster=cluster2 2026-02-22T21:37:34.804 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local uuid_var_name 2026-02-22T21:37:34.804 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n '' ']' 2026-02-22T21:37:34.804 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local error_code 2026-02-22T21:37:34.804 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local peer_uuid 2026-02-22T21:37:34.804 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 16 32 2026-02-22T21:37:34.804 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set +e 2026-02-22T21:37:34.804 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster cluster1 mirror pool peer add mirror_parent cluster2 2026-02-22T21:37:34.826 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ peer_uuid=858a8194-1f8e-4771-b3b3-76bd2e64c25d 2026-02-22T21:37:34.826 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ error_code=0 2026-02-22T21:37:34.826 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set -e 2026-02-22T21:37:34.826 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' 0 -eq 17 ']' 2026-02-22T21:37:34.826 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 0 -eq 0 2026-02-22T21:37:34.826 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n '' ']' 2026-02-22T21:37:34.826 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:37:34.826 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ setup_pools cluster2 cluster1 2026-02-22T21:37:34.826 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:37:34.826 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local remote_cluster=cluster1 2026-02-22T21:37:34.826 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mon_map_file 2026-02-22T21:37:34.826 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mon_addr 2026-02-22T21:37:34.826 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local admin_key_file 2026-02-22T21:37:34.826 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local uuid 2026-02-22T21:37:34.826 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ CEPH_ARGS= 2026-02-22T21:37:34.827 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ ceph --cluster cluster2 osd pool create mirror 64 64 2026-02-22T21:37:35.128 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:pool 'mirror' already exists 2026-02-22T21:37:35.138 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ CEPH_ARGS= 2026-02-22T21:37:35.138 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ ceph --cluster cluster2 osd pool create mirror_parent 64 64 2026-02-22T21:37:36.139 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:pool 'mirror_parent' already exists 2026-02-22T21:37:36.147 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ CEPH_ARGS= 2026-02-22T21:37:36.147 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 pool init mirror 2026-02-22T21:37:39.101 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ CEPH_ARGS= 2026-02-22T21:37:39.102 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 pool init mirror_parent 2026-02-22T21:37:42.105 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n '' ']' 2026-02-22T21:37:42.105 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ CEPH_ARGS= 2026-02-22T21:37:42.105 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 mirror pool enable --site-name cluster2 mirror image 2026-02-22T21:37:42.132 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 mirror pool enable mirror_parent image 2026-02-22T21:37:42.155 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 namespace create mirror/ns1 2026-02-22T21:37:42.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 namespace create mirror/ns2 2026-02-22T21:37:42.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 namespace create mirror_parent/ns1 2026-02-22T21:37:42.425 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 mirror pool enable mirror/ns1 image 2026-02-22T21:37:42.449 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 mirror pool enable mirror/ns2 image 2026-02-22T21:37:42.473 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 mirror pool enable mirror_parent/ns1 image 2026-02-22T21:37:42.496 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -z ']' 2026-02-22T21:37:42.496 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -z ']' 2026-02-22T21:37:42.496 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ peer_add cluster2 mirror cluster1 2026-02-22T21:37:42.496 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:37:42.496 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:37:42.496 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:37:42.496 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:37:42.496 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local client_cluster=cluster1 2026-02-22T21:37:42.496 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:37:42.496 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local remote_cluster=cluster1 2026-02-22T21:37:42.496 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local uuid_var_name 2026-02-22T21:37:42.496 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n '' ']' 2026-02-22T21:37:42.496 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local error_code 2026-02-22T21:37:42.496 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local peer_uuid 2026-02-22T21:37:42.496 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 16 32 2026-02-22T21:37:42.496 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set +e 2026-02-22T21:37:42.496 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster cluster2 mirror pool peer add mirror cluster1 2026-02-22T21:37:42.519 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ peer_uuid=5e3bdadc-1227-45f6-b318-5cb0390ccd5b 2026-02-22T21:37:42.519 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ error_code=0 2026-02-22T21:37:42.519 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set -e 2026-02-22T21:37:42.519 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' 0 -eq 17 ']' 2026-02-22T21:37:42.519 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 0 -eq 0 2026-02-22T21:37:42.519 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n '' ']' 2026-02-22T21:37:42.519 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:37:42.519 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ peer_add cluster2 mirror_parent cluster1 2026-02-22T21:37:42.519 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:37:42.519 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:37:42.519 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror_parent 2026-02-22T21:37:42.519 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:37:42.519 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local client_cluster=cluster1 2026-02-22T21:37:42.519 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:37:42.519 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local remote_cluster=cluster1 2026-02-22T21:37:42.519 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local uuid_var_name 2026-02-22T21:37:42.519 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n '' ']' 2026-02-22T21:37:42.519 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local error_code 2026-02-22T21:37:42.519 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local peer_uuid 2026-02-22T21:37:42.519 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 16 32 2026-02-22T21:37:42.519 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set +e 2026-02-22T21:37:42.519 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster cluster2 mirror pool peer add mirror_parent cluster1 2026-02-22T21:37:42.542 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ peer_uuid=e4557e69-4e7d-474b-9bb2-83ba24614428 2026-02-22T21:37:42.542 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ error_code=0 2026-02-22T21:37:42.542 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set -e 2026-02-22T21:37:42.542 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' 0 -eq 17 ']' 2026-02-22T21:37:42.542 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 0 -eq 0 2026-02-22T21:37:42.542 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n '' ']' 2026-02-22T21:37:42.542 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:37:42.542 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n '' ']' 2026-02-22T21:37:42.542 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ start_mirrors cluster1 2026-02-22T21:37:42.542 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:37:42.542 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ seq 0 3 2026-02-22T21:37:42.543 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:37:42.543 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ start_mirror cluster1:0 2026-02-22T21:37:42.543 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1:0 2026-02-22T21:37:42.544 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local instance 2026-02-22T21:37:42.544 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:37:42.544 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local val=cluster1:0 2026-02-22T21:37:42.544 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster_var_name=cluster 2026-02-22T21:37:42.544 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local instance_var_name=instance 2026-02-22T21:37:42.544 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ cluster=cluster1 2026-02-22T21:37:42.544 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ instance=0 2026-02-22T21:37:42.544 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T21:37:42.544 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ eval cluster=cluster1 2026-02-22T21:37:42.544 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:37:42.544 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ eval instance=0 2026-02-22T21:37:42.544 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=0 2026-02-22T21:37:42.544 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n 1 2026-02-22T21:37:42.544 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 2026-02-22T21:37:42.544 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:37:42.544 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ start_mirror cluster1:1 2026-02-22T21:37:42.544 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1:1 2026-02-22T21:37:42.544 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local instance 2026-02-22T21:37:42.545 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T21:37:42.545 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local val=cluster1:1 2026-02-22T21:37:42.545 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster_var_name=cluster 2026-02-22T21:37:42.545 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local instance_var_name=instance 2026-02-22T21:37:42.545 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ cluster=cluster1 2026-02-22T21:37:42.545 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ instance=1 2026-02-22T21:37:42.545 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T21:37:42.545 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ eval cluster=cluster1 2026-02-22T21:37:42.545 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:37:42.545 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ eval instance=1 2026-02-22T21:37:42.545 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=1 2026-02-22T21:37:42.545 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n 1 2026-02-22T21:37:42.545 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 2026-02-22T21:37:42.545 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:37:42.545 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ start_mirror cluster1:2 2026-02-22T21:37:42.545 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1:2 2026-02-22T21:37:42.545 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local instance 2026-02-22T21:37:42.545 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-22T21:37:42.545 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local val=cluster1:2 2026-02-22T21:37:42.545 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster_var_name=cluster 2026-02-22T21:37:42.545 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local instance_var_name=instance 2026-02-22T21:37:42.545 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ cluster=cluster1 2026-02-22T21:37:42.545 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ instance=2 2026-02-22T21:37:42.545 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-22T21:37:42.545 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ eval cluster=cluster1 2026-02-22T21:37:42.545 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:37:42.545 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ eval instance=2 2026-02-22T21:37:42.545 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=2 2026-02-22T21:37:42.545 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n 1 2026-02-22T21:37:42.545 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 2026-02-22T21:37:42.545 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:37:42.545 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ start_mirror cluster1:3 2026-02-22T21:37:42.545 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1:3 2026-02-22T21:37:42.545 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local instance 2026-02-22T21:37:42.545 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-22T21:37:42.545 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local val=cluster1:3 2026-02-22T21:37:42.545 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster_var_name=cluster 2026-02-22T21:37:42.545 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local instance_var_name=instance 2026-02-22T21:37:42.545 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ cluster=cluster1 2026-02-22T21:37:42.545 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ instance=3 2026-02-22T21:37:42.545 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-22T21:37:42.546 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ eval cluster=cluster1 2026-02-22T21:37:42.546 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:37:42.546 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ eval instance=3 2026-02-22T21:37:42.546 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=3 2026-02-22T21:37:42.546 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n 1 2026-02-22T21:37:42.546 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 2026-02-22T21:37:42.546 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ start_mirrors cluster2 2026-02-22T21:37:42.546 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:37:42.546 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ seq 0 3 2026-02-22T21:37:42.547 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:37:42.547 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ start_mirror cluster2:0 2026-02-22T21:37:42.548 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2:0 2026-02-22T21:37:42.548 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local instance 2026-02-22T21:37:42.548 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:37:42.548 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local val=cluster2:0 2026-02-22T21:37:42.548 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster_var_name=cluster 2026-02-22T21:37:42.548 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local instance_var_name=instance 2026-02-22T21:37:42.548 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ cluster=cluster2 2026-02-22T21:37:42.548 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ instance=0 2026-02-22T21:37:42.548 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' 0 = cluster2:0 ']' 2026-02-22T21:37:42.548 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ eval cluster=cluster2 2026-02-22T21:37:42.548 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster2 2026-02-22T21:37:42.548 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ eval instance=0 2026-02-22T21:37:42.548 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=0 2026-02-22T21:37:42.548 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n 1 2026-02-22T21:37:42.548 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 2026-02-22T21:37:42.548 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:37:42.548 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ start_mirror cluster2:1 2026-02-22T21:37:42.548 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2:1 2026-02-22T21:37:42.548 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local instance 2026-02-22T21:37:42.548 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set_cluster_instance cluster2:1 cluster instance 2026-02-22T21:37:42.548 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local val=cluster2:1 2026-02-22T21:37:42.548 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster_var_name=cluster 2026-02-22T21:37:42.548 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local instance_var_name=instance 2026-02-22T21:37:42.548 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ cluster=cluster2 2026-02-22T21:37:42.548 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ instance=1 2026-02-22T21:37:42.548 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' 1 = cluster2:1 ']' 2026-02-22T21:37:42.548 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ eval cluster=cluster2 2026-02-22T21:37:42.548 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster2 2026-02-22T21:37:42.548 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ eval instance=1 2026-02-22T21:37:42.548 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=1 2026-02-22T21:37:42.548 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n 1 2026-02-22T21:37:42.548 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 2026-02-22T21:37:42.548 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:37:42.548 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ start_mirror cluster2:2 2026-02-22T21:37:42.548 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2:2 2026-02-22T21:37:42.548 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local instance 2026-02-22T21:37:42.548 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set_cluster_instance cluster2:2 cluster instance 2026-02-22T21:37:42.548 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local val=cluster2:2 2026-02-22T21:37:42.548 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster_var_name=cluster 2026-02-22T21:37:42.548 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local instance_var_name=instance 2026-02-22T21:37:42.549 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ cluster=cluster2 2026-02-22T21:37:42.549 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ instance=2 2026-02-22T21:37:42.549 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' 2 = cluster2:2 ']' 2026-02-22T21:37:42.549 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ eval cluster=cluster2 2026-02-22T21:37:42.549 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster2 2026-02-22T21:37:42.549 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ eval instance=2 2026-02-22T21:37:42.549 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=2 2026-02-22T21:37:42.549 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n 1 2026-02-22T21:37:42.549 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 2026-02-22T21:37:42.549 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:37:42.549 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ start_mirror cluster2:3 2026-02-22T21:37:42.549 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2:3 2026-02-22T21:37:42.549 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local instance 2026-02-22T21:37:42.549 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set_cluster_instance cluster2:3 cluster instance 2026-02-22T21:37:42.549 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local val=cluster2:3 2026-02-22T21:37:42.549 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster_var_name=cluster 2026-02-22T21:37:42.549 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local instance_var_name=instance 2026-02-22T21:37:42.549 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ cluster=cluster2 2026-02-22T21:37:42.549 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ instance=3 2026-02-22T21:37:42.549 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' 3 = cluster2:3 ']' 2026-02-22T21:37:42.549 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ eval cluster=cluster2 2026-02-22T21:37:42.549 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster2 2026-02-22T21:37:42.549 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ eval instance=3 2026-02-22T21:37:42.549 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=3 2026-02-22T21:37:42.549 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n 1 2026-02-22T21:37:42.549 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 2026-02-22T21:37:42.549 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ testlog 'TEST: add image and test replay after client crashes' 2026-02-22T21:37:42.549 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ tee -a /tmp/tmp.j44LZPowal/rbd-mirror.test.log 2026-02-22T21:37:42.549 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ date '+%F %T' 2026-02-22T21:37:42.549 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ echo 2026-02-22 21:37:42 TEST: add image and test replay after client crashes 2026-02-22T21:37:42.549 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:2026-02-22 21:37:42 TEST: add image and test replay after client crashes 2026-02-22T21:37:42.549 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=test 2026-02-22T21:37:42.549 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image_and_enable_mirror cluster2 mirror test snapshot 512M 2026-02-22T21:37:42.549 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:37:42.549 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:37:42.549 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:37:42.549 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:37:42.549 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:37:42.549 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:37:42.549 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T21:37:42.549 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n snapshot ']' 2026-02-22T21:37:42.550 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:37:42.550 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image cluster2 mirror test 512M 2026-02-22T21:37:42.550 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:37:42.550 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:37:42.550 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:37:42.550 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:37:42.550 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:37:42.550 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:37:42.550 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local size=128 2026-02-22T21:37:42.550 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n 512M ']' 2026-02-22T21:37:42.550 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ size=512M 2026-02-22T21:37:42.550 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:37:42.550 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering,exclusive-lock mirror/test 2026-02-22T21:37:42.574 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T21:37:42.574 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:37:42.574 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' image = image ']' 2026-02-22T21:37:42.574 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:37:42.574 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ enable_mirror cluster2 mirror test snapshot 2026-02-22T21:37:42.574 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:37:42.574 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:37:42.574 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:37:42.574 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T21:37:42.574 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/test snapshot 2026-02-22T21:37:43.139 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Mirroring enabled 2026-02-22T21:37:43.144 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 info mirror/test 2026-02-22T21:37:43.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'test': 2026-02-22T21:37:43.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 512 MiB in 128 objects 2026-02-22T21:37:43.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T21:37:43.169 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 1 2026-02-22T21:37:43.169 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 10af5d44aa45 2026-02-22T21:37:43.169 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.10af5d44aa45 2026-02-22T21:37:43.169 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T21:37:43.169 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock 2026-02-22T21:37:43.169 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T21:37:43.169 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T21:37:43.169 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:37:42 2026 2026-02-22T21:37:43.169 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:37:42 2026 2026-02-22T21:37:43.169 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:37:42 2026 2026-02-22T21:37:43.169 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T21:37:43.170 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T21:37:43.170 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: b410963b-cae9-4337-a6ef-da65321c9374 2026-02-22T21:37:43.170 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: true 2026-02-22T21:37:43.171 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T21:37:43.171 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:37:43.171 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:37:43.171 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:37:43.171 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T21:37:43.171 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:37:43.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:37:43.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:37:43.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=started 2026-02-22T21:37:43.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T21:37:43.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:37:43.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T21:37:44.171 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T21:37:44.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:37:44.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:37:44.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:37:44.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=started 2026-02-22T21:37:44.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_result 2026-02-22T21:37:44.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=stopped 2026-02-22T21:37:44.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:37:44.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_instance=cluster1 2026-02-22T21:37:44.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ grep -i state 2026-02-22T21:37:44.173 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:37:44.173 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T21:37:44.173 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance=cluster1 2026-02-22T21:37:44.173 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local loop_instance 2026-02-22T21:37:44.173 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:37:44.173 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sleep 0 2026-02-22T21:37:44.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:37:44.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ seq 0 3 2026-02-22T21:37:44.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:37:44.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:37:44.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:0 2026-02-22T21:37:44.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:37:44.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:37:44.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:37:44.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:0 2026-02-22T21:37:44.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:37:44.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:37:44.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:37:44.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=0 2026-02-22T21:37:44.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:37:44.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:37:44.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:37:44.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=0 2026-02-22T21:37:44.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T21:37:44.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:37:44.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:37:44.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:37:44.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:37:44.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:37:44.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:0 2026-02-22T21:37:44.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:37:44.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:37:44.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:37:44.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T21:37:44.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:37:44.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:37:44.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:37:44.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=0 2026-02-22T21:37:44.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=0 2026-02-22T21:37:44.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:37:44.187 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:37:44.188 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:37:44.188 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:37:44.188 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:37:44.227 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T21:37:44.227 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:37:44.227 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T21:37:44.230 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:37:44.230 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-22T21:37:44.230 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:1 2026-02-22T21:37:44.230 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:37:44.230 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:37:44.230 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T21:37:44.230 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:1 2026-02-22T21:37:44.230 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:37:44.230 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:37:44.230 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:37:44.230 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=1 2026-02-22T21:37:44.230 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T21:37:44.230 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:37:44.230 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:37:44.230 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=1 2026-02-22T21:37:44.230 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T21:37:44.230 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T21:37:44.230 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:1 2026-02-22T21:37:44.230 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:37:44.230 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:37:44.230 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T21:37:44.230 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:1 2026-02-22T21:37:44.230 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:37:44.230 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:37:44.230 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:37:44.230 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T21:37:44.230 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T21:37:44.230 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:37:44.230 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:37:44.231 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=1 2026-02-22T21:37:44.231 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=1 2026-02-22T21:37:44.231 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T21:37:44.242 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:37:44.242 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:37:44.242 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:37:44.242 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-22T21:37:44.282 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T21:37:44.283 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:37:44.283 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T21:37:44.286 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:37:44.286 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2026-02-22T21:37:44.286 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:2 2026-02-22T21:37:44.286 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:37:44.287 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:37:44.287 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T21:37:44.287 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:2 2026-02-22T21:37:44.287 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:37:44.287 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:37:44.287 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:37:44.287 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=2 2026-02-22T21:37:44.287 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T21:37:44.287 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:37:44.287 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:37:44.287 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=2 2026-02-22T21:37:44.287 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T21:37:44.288 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T21:37:44.288 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:2 2026-02-22T21:37:44.288 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:37:44.288 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:37:44.288 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T21:37:44.288 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:2 2026-02-22T21:37:44.288 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:37:44.288 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:37:44.288 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:37:44.289 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T21:37:44.289 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T21:37:44.289 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:37:44.289 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:37:44.289 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=2 2026-02-22T21:37:44.289 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=2 2026-02-22T21:37:44.289 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T21:37:44.296 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:37:44.296 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:37:44.296 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:37:44.296 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2026-02-22T21:37:44.335 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T21:37:44.335 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:37:44.335 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T21:37:44.338 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:37:44.338 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2026-02-22T21:37:44.339 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:3 2026-02-22T21:37:44.339 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:37:44.339 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:37:44.339 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T21:37:44.339 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:3 2026-02-22T21:37:44.339 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:37:44.339 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:37:44.339 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:37:44.339 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=3 2026-02-22T21:37:44.339 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T21:37:44.339 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:37:44.339 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:37:44.340 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=3 2026-02-22T21:37:44.340 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=3 2026-02-22T21:37:44.340 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T21:37:44.340 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:3 2026-02-22T21:37:44.340 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:37:44.340 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:37:44.340 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T21:37:44.340 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:3 2026-02-22T21:37:44.340 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:37:44.340 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:37:44.340 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:37:44.340 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=3 2026-02-22T21:37:44.341 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T21:37:44.341 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:37:44.341 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:37:44.341 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=3 2026-02-22T21:37:44.341 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=3 2026-02-22T21:37:44.341 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T21:37:44.349 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:37:44.350 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:37:44.350 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:37:44.350 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2026-02-22T21:37:44.388 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T21:37:44.388 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:37:44.388 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T21:37:44.389 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:37:44.389 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sleep 1 2026-02-22T21:37:45.389 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:37:45.390 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ seq 0 3 2026-02-22T21:37:45.392 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:37:45.392 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:37:45.392 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:0 2026-02-22T21:37:45.392 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:37:45.392 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:37:45.392 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:37:45.392 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:0 2026-02-22T21:37:45.392 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:37:45.392 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:37:45.393 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:37:45.393 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=0 2026-02-22T21:37:45.393 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:37:45.393 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:37:45.393 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:37:45.393 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=0 2026-02-22T21:37:45.393 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T21:37:45.393 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:37:45.393 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:37:45.393 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:37:45.393 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:37:45.393 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:37:45.393 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:0 2026-02-22T21:37:45.393 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:37:45.393 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:37:45.393 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:37:45.393 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T21:37:45.393 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:37:45.393 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:37:45.393 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:37:45.393 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=0 2026-02-22T21:37:45.394 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=0 2026-02-22T21:37:45.394 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:37:45.405 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:37:45.406 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:37:45.406 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:37:45.406 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:37:45.446 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T21:37:45.446 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:37:45.446 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T21:37:45.447 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:37:45.447 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-22T21:37:45.447 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:1 2026-02-22T21:37:45.447 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:37:45.447 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:37:45.447 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T21:37:45.447 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:1 2026-02-22T21:37:45.447 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:37:45.448 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:37:45.448 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:37:45.448 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=1 2026-02-22T21:37:45.448 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T21:37:45.448 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:37:45.448 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:37:45.448 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=1 2026-02-22T21:37:45.448 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T21:37:45.448 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T21:37:45.448 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:1 2026-02-22T21:37:45.448 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:37:45.448 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:37:45.448 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T21:37:45.448 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:1 2026-02-22T21:37:45.448 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:37:45.448 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:37:45.448 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:37:45.448 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T21:37:45.448 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T21:37:45.448 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:37:45.448 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:37:45.448 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=1 2026-02-22T21:37:45.449 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=1 2026-02-22T21:37:45.449 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T21:37:45.460 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:37:45.460 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:37:45.460 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:37:45.461 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-22T21:37:45.501 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T21:37:45.501 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:37:45.501 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T21:37:45.504 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:37:45.504 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2026-02-22T21:37:45.504 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:2 2026-02-22T21:37:45.504 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:37:45.504 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:37:45.504 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T21:37:45.504 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:2 2026-02-22T21:37:45.504 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:37:45.504 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:37:45.504 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:37:45.505 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=2 2026-02-22T21:37:45.505 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T21:37:45.505 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:37:45.505 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:37:45.505 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=2 2026-02-22T21:37:45.505 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T21:37:45.505 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T21:37:45.505 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:2 2026-02-22T21:37:45.505 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:37:45.505 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:37:45.505 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T21:37:45.505 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:2 2026-02-22T21:37:45.506 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:37:45.506 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:37:45.506 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:37:45.506 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T21:37:45.506 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T21:37:45.506 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:37:45.506 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:37:45.506 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=2 2026-02-22T21:37:45.506 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=2 2026-02-22T21:37:45.506 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T21:37:45.514 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:37:45.514 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:37:45.515 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:37:45.515 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2026-02-22T21:37:45.552 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T21:37:45.552 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:37:45.552 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T21:37:45.553 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:37:45.554 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2026-02-22T21:37:45.554 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:3 2026-02-22T21:37:45.554 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:37:45.554 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:37:45.554 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T21:37:45.554 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:3 2026-02-22T21:37:45.554 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:37:45.554 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:37:45.554 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:37:45.554 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=3 2026-02-22T21:37:45.554 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T21:37:45.554 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:37:45.554 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:37:45.554 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=3 2026-02-22T21:37:45.554 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=3 2026-02-22T21:37:45.554 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T21:37:45.554 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:3 2026-02-22T21:37:45.554 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:37:45.554 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:37:45.554 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T21:37:45.554 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:3 2026-02-22T21:37:45.554 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:37:45.554 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:37:45.555 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:37:45.555 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=3 2026-02-22T21:37:45.555 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T21:37:45.555 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:37:45.555 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:37:45.555 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=3 2026-02-22T21:37:45.555 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=3 2026-02-22T21:37:45.555 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T21:37:45.565 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:37:45.565 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:37:45.565 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:37:45.565 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2026-02-22T21:37:45.606 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T21:37:45.607 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:37:45.607 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T21:37:45.608 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:37:45.608 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sleep 2 2026-02-22T21:37:47.608 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:37:47.609 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ seq 0 3 2026-02-22T21:37:47.612 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:37:47.612 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:37:47.612 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:0 2026-02-22T21:37:47.612 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:37:47.612 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:37:47.612 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:37:47.613 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:0 2026-02-22T21:37:47.613 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:37:47.613 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:37:47.613 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:37:47.613 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=0 2026-02-22T21:37:47.613 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:37:47.613 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:37:47.613 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:37:47.613 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=0 2026-02-22T21:37:47.613 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T21:37:47.613 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:37:47.613 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:37:47.613 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:37:47.613 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:37:47.614 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:37:47.614 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:0 2026-02-22T21:37:47.614 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:37:47.614 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:37:47.614 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:37:47.614 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T21:37:47.614 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:37:47.614 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:37:47.614 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:37:47.614 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=0 2026-02-22T21:37:47.614 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=0 2026-02-22T21:37:47.614 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:37:47.623 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:37:47.624 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:37:47.624 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:37:47.624 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:37:47.663 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T21:37:47.663 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:37:47.663 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T21:37:47.666 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:37:47.666 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-22T21:37:47.666 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:1 2026-02-22T21:37:47.666 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:37:47.666 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:37:47.666 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T21:37:47.667 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:1 2026-02-22T21:37:47.667 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:37:47.667 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:37:47.667 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:37:47.667 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=1 2026-02-22T21:37:47.667 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T21:37:47.667 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:37:47.667 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:37:47.667 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=1 2026-02-22T21:37:47.667 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T21:37:47.667 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T21:37:47.667 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:1 2026-02-22T21:37:47.668 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:37:47.668 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:37:47.668 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T21:37:47.668 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:1 2026-02-22T21:37:47.668 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:37:47.668 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:37:47.668 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:37:47.668 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T21:37:47.668 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T21:37:47.668 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:37:47.668 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:37:47.668 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=1 2026-02-22T21:37:47.669 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=1 2026-02-22T21:37:47.669 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T21:37:47.676 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:37:47.676 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:37:47.676 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:37:47.677 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-22T21:37:47.716 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T21:37:47.716 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:37:47.716 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T21:37:47.718 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:37:47.718 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2026-02-22T21:37:47.718 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:2 2026-02-22T21:37:47.718 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:37:47.719 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:37:47.719 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T21:37:47.719 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:2 2026-02-22T21:37:47.719 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:37:47.719 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:37:47.719 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:37:47.719 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=2 2026-02-22T21:37:47.719 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T21:37:47.719 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:37:47.719 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:37:47.719 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=2 2026-02-22T21:37:47.719 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T21:37:47.719 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T21:37:47.719 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:2 2026-02-22T21:37:47.719 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:37:47.719 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:37:47.719 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T21:37:47.719 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:2 2026-02-22T21:37:47.719 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:37:47.719 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:37:47.719 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:37:47.719 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T21:37:47.719 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T21:37:47.719 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:37:47.719 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:37:47.720 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=2 2026-02-22T21:37:47.720 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=2 2026-02-22T21:37:47.720 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T21:37:47.730 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:37:47.730 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:37:47.730 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:37:47.730 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2026-02-22T21:37:47.769 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T21:37:47.770 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:37:47.770 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T21:37:47.771 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:37:47.771 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2026-02-22T21:37:47.771 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:3 2026-02-22T21:37:47.771 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:37:47.772 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:37:47.772 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T21:37:47.772 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:3 2026-02-22T21:37:47.772 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:37:47.772 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:37:47.772 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:37:47.772 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=3 2026-02-22T21:37:47.772 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T21:37:47.772 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:37:47.772 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:37:47.772 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=3 2026-02-22T21:37:47.772 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=3 2026-02-22T21:37:47.772 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T21:37:47.772 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:3 2026-02-22T21:37:47.772 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:37:47.772 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:37:47.772 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T21:37:47.772 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:3 2026-02-22T21:37:47.772 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:37:47.772 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:37:47.772 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:37:47.772 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=3 2026-02-22T21:37:47.772 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T21:37:47.772 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:37:47.772 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:37:47.772 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=3 2026-02-22T21:37:47.772 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=3 2026-02-22T21:37:47.772 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T21:37:47.783 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:37:47.783 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:37:47.783 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:37:47.783 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2026-02-22T21:37:47.822 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T21:37:47.822 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:37:47.822 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T21:37:47.823 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:37:47.823 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sleep 4 2026-02-22T21:37:51.824 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:37:51.824 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ seq 0 3 2026-02-22T21:37:51.826 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:37:51.826 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:37:51.826 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:0 2026-02-22T21:37:51.826 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:37:51.826 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:37:51.826 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:37:51.826 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:0 2026-02-22T21:37:51.826 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:37:51.826 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:37:51.826 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:37:51.826 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=0 2026-02-22T21:37:51.826 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:37:51.826 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:37:51.826 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:37:51.826 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=0 2026-02-22T21:37:51.826 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T21:37:51.827 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:37:51.827 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:37:51.827 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:37:51.827 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:37:51.827 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:37:51.827 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:0 2026-02-22T21:37:51.827 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:37:51.827 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:37:51.827 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:37:51.827 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T21:37:51.827 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:37:51.827 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:37:51.827 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:37:51.827 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=0 2026-02-22T21:37:51.827 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=0 2026-02-22T21:37:51.827 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:37:51.840 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:37:51.840 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:37:51.840 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:37:51.840 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:37:51.881 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T21:37:51.881 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:37:51.881 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T21:37:51.883 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:37:51.883 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-22T21:37:51.883 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:1 2026-02-22T21:37:51.883 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:37:51.883 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:37:51.883 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T21:37:51.883 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:1 2026-02-22T21:37:51.883 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:37:51.883 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:37:51.883 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:37:51.883 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=1 2026-02-22T21:37:51.883 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T21:37:51.883 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:37:51.883 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:37:51.883 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=1 2026-02-22T21:37:51.883 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T21:37:51.883 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T21:37:51.883 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:1 2026-02-22T21:37:51.883 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:37:51.883 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:37:51.883 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T21:37:51.883 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:1 2026-02-22T21:37:51.884 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:37:51.884 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:37:51.884 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:37:51.884 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T21:37:51.884 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T21:37:51.884 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:37:51.884 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:37:51.884 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=1 2026-02-22T21:37:51.884 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=1 2026-02-22T21:37:51.884 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T21:37:51.895 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:37:51.896 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:37:51.896 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:37:51.896 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-22T21:37:51.937 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T21:37:51.937 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:37:51.937 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T21:37:51.940 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:37:51.940 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2026-02-22T21:37:51.940 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:2 2026-02-22T21:37:51.940 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:37:51.940 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:37:51.940 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T21:37:51.940 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:2 2026-02-22T21:37:51.941 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:37:51.941 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:37:51.941 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:37:51.941 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=2 2026-02-22T21:37:51.941 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T21:37:51.941 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:37:51.941 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:37:51.941 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=2 2026-02-22T21:37:51.941 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T21:37:51.941 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T21:37:51.941 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:2 2026-02-22T21:37:51.941 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:37:51.942 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:37:51.942 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T21:37:51.942 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:2 2026-02-22T21:37:51.942 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:37:51.942 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:37:51.942 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:37:51.942 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T21:37:51.942 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T21:37:51.942 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:37:51.942 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:37:51.942 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=2 2026-02-22T21:37:51.942 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=2 2026-02-22T21:37:51.943 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T21:37:51.951 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:37:51.951 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:37:51.951 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:37:51.951 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2026-02-22T21:37:51.991 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T21:37:51.991 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:37:51.991 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T21:37:51.994 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:37:51.994 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2026-02-22T21:37:51.994 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:3 2026-02-22T21:37:51.994 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:37:51.995 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:37:51.995 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T21:37:51.995 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:3 2026-02-22T21:37:51.995 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:37:51.995 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:37:51.995 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:37:51.995 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=3 2026-02-22T21:37:51.995 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T21:37:51.995 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:37:51.995 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:37:51.995 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=3 2026-02-22T21:37:51.996 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=3 2026-02-22T21:37:51.996 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T21:37:51.996 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:3 2026-02-22T21:37:51.996 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:37:51.996 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:37:51.996 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T21:37:51.996 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:3 2026-02-22T21:37:51.996 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:37:51.996 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:37:51.996 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:37:51.996 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=3 2026-02-22T21:37:51.996 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T21:37:51.997 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:37:51.997 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:37:51.997 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=3 2026-02-22T21:37:51.997 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=3 2026-02-22T21:37:51.997 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T21:37:52.004 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:37:52.004 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:37:52.004 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:37:52.004 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2026-02-22T21:37:52.044 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T21:37:52.045 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:37:52.045 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T21:37:52.046 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:37:52.046 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sleep 8 2026-02-22T21:38:00.047 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:38:00.048 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ seq 0 3 2026-02-22T21:38:00.050 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:38:00.051 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:38:00.051 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:0 2026-02-22T21:38:00.051 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:38:00.051 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:38:00.051 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:38:00.051 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:0 2026-02-22T21:38:00.051 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:38:00.051 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:38:00.051 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:38:00.051 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=0 2026-02-22T21:38:00.052 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:38:00.052 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:38:00.052 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:38:00.052 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=0 2026-02-22T21:38:00.052 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T21:38:00.052 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:38:00.052 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:38:00.052 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:38:00.052 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:38:00.052 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:38:00.052 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:0 2026-02-22T21:38:00.052 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:38:00.053 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:38:00.053 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:38:00.053 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T21:38:00.053 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:38:00.053 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:38:00.053 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:38:00.053 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=0 2026-02-22T21:38:00.053 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=0 2026-02-22T21:38:00.053 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:38:00.064 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:38:00.064 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:38:00.064 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:38:00.064 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:38:00.104 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T21:38:00.104 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:38:00.104 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T21:38:00.107 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:38:00.107 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-22T21:38:00.107 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:1 2026-02-22T21:38:00.107 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:38:00.108 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:38:00.108 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T21:38:00.108 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:1 2026-02-22T21:38:00.108 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:38:00.108 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:38:00.108 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:38:00.108 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=1 2026-02-22T21:38:00.108 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T21:38:00.108 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:38:00.109 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:38:00.109 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=1 2026-02-22T21:38:00.109 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T21:38:00.109 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T21:38:00.109 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:1 2026-02-22T21:38:00.109 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:38:00.109 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:38:00.109 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T21:38:00.109 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:1 2026-02-22T21:38:00.109 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:38:00.109 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:38:00.110 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:38:00.110 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T21:38:00.110 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T21:38:00.110 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:38:00.110 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:38:00.110 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=1 2026-02-22T21:38:00.110 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=1 2026-02-22T21:38:00.110 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T21:38:00.120 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:38:00.120 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:38:00.120 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:38:00.120 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-22T21:38:00.159 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T21:38:00.160 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:38:00.160 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T21:38:00.162 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:38:00.163 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2026-02-22T21:38:00.163 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:2 2026-02-22T21:38:00.163 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:38:00.163 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:38:00.163 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T21:38:00.163 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:2 2026-02-22T21:38:00.163 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:38:00.163 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:38:00.163 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:38:00.163 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=2 2026-02-22T21:38:00.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T21:38:00.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:38:00.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:38:00.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=2 2026-02-22T21:38:00.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T21:38:00.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T21:38:00.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:2 2026-02-22T21:38:00.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:38:00.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:38:00.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T21:38:00.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:2 2026-02-22T21:38:00.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:38:00.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:38:00.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:38:00.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T21:38:00.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T21:38:00.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:38:00.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:38:00.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=2 2026-02-22T21:38:00.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=2 2026-02-22T21:38:00.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T21:38:00.173 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:38:00.173 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:38:00.173 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:38:00.173 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2026-02-22T21:38:00.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T21:38:00.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:38:00.213 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T21:38:00.214 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:38:00.214 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2026-02-22T21:38:00.215 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:3 2026-02-22T21:38:00.215 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:38:00.215 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:38:00.215 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T21:38:00.215 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:3 2026-02-22T21:38:00.215 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:38:00.215 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:38:00.215 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:38:00.215 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=3 2026-02-22T21:38:00.215 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T21:38:00.215 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:38:00.215 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:38:00.215 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=3 2026-02-22T21:38:00.215 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=3 2026-02-22T21:38:00.215 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T21:38:00.215 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:3 2026-02-22T21:38:00.215 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:38:00.215 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:38:00.215 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T21:38:00.215 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:3 2026-02-22T21:38:00.215 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:38:00.215 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:38:00.215 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:38:00.215 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=3 2026-02-22T21:38:00.215 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T21:38:00.215 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:38:00.215 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:38:00.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=3 2026-02-22T21:38:00.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=3 2026-02-22T21:38:00.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T21:38:00.227 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:38:00.227 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:38:00.227 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:38:00.227 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2026-02-22T21:38:00.268 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T21:38:00.268 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:38:00.268 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T21:38:00.269 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:38:00.269 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sleep 8 2026-02-22T21:38:08.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:38:08.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ seq 0 3 2026-02-22T21:38:08.274 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:38:08.274 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:38:08.274 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:0 2026-02-22T21:38:08.274 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:38:08.274 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:38:08.274 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:38:08.274 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:0 2026-02-22T21:38:08.275 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:38:08.275 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:38:08.275 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:38:08.275 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=0 2026-02-22T21:38:08.275 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:38:08.275 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:38:08.275 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:38:08.275 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=0 2026-02-22T21:38:08.275 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T21:38:08.275 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:38:08.275 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:38:08.275 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:38:08.276 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:38:08.276 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:38:08.276 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:0 2026-02-22T21:38:08.276 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:38:08.276 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:38:08.276 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:38:08.276 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T21:38:08.276 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:38:08.276 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:38:08.276 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:38:08.277 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=0 2026-02-22T21:38:08.277 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=0 2026-02-22T21:38:08.277 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:38:08.285 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:38:08.286 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:38:08.286 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:38:08.286 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:38:08.332 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ return 0 2026-02-22T21:38:08.332 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:38:08.333 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:38:08.333 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep -i Replaying 2026-02-22T21:38:08.334 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: "state": "Replaying" 2026-02-22T21:38:08.334 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_state=started 2026-02-22T21:38:08.334 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test started = started 2026-02-22T21:38:08.335 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:38:08.335 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ clean_snap_name= 2026-02-22T21:38:08.335 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ seq 1 10 2026-02-22T21:38:08.335 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 10` 2026-02-22T21:38:08.336 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ stress_write_image cluster2 mirror test 2026-02-22T21:38:08.336 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:38:08.336 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:38:08.336 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:38:08.336 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-22T21:38:08.337 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local duration=6 2026-02-22T21:38:08.337 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set +e 2026-02-22T21:38:08.337 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ timeout 6s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2026-02-22T21:38:08.403 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:100 IOs, 47448064 bytes 2026-02-22T21:38:08.423 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:200 IOs, 71619584 bytes 2026-02-22T21:38:08.439 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:300 IOs, 82840576 bytes 2026-02-22T21:38:08.463 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:400 IOs, 105808896 bytes 2026-02-22T21:38:08.492 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:500 IOs, 155191296 bytes 2026-02-22T21:38:08.506 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:600 IOs, 155830272 bytes 2026-02-22T21:38:08.538 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:700 IOs, 207642624 bytes 2026-02-22T21:38:08.568 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:800 IOs, 260541440 bytes 2026-02-22T21:38:08.592 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:900 IOs, 286146560 bytes 2026-02-22T21:38:08.619 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1000 IOs, 316749824 bytes 2026-02-22T21:38:08.639 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1100 IOs, 335766528 bytes 2026-02-22T21:38:08.665 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1200 IOs, 361777152 bytes 2026-02-22T21:38:08.730 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1300 IOs, 502705152 bytes 2026-02-22T21:38:08.766 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1400 IOs, 558273536 bytes 2026-02-22T21:38:08.797 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1500 IOs, 617958400 bytes 2026-02-22T21:38:08.825 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1600 IOs, 638519296 bytes 2026-02-22T21:38:08.850 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1700 IOs, 666808320 bytes 2026-02-22T21:38:08.876 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1800 IOs, 694648832 bytes 2026-02-22T21:38:08.901 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1900 IOs, 723695616 bytes 2026-02-22T21:38:08.933 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2000 IOs, 761432064 bytes 2026-02-22T21:38:08.968 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2100 IOs, 813376512 bytes 2026-02-22T21:38:09.005 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2200 IOs, 876950528 bytes 2026-02-22T21:38:09.045 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2300 IOs, 961939456 bytes 2026-02-22T21:38:09.098 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2400 IOs, 1022767104 bytes 2026-02-22T21:38:09.142 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2500 IOs, 1100711936 bytes 2026-02-22T21:38:09.180 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2600 IOs, 1150883840 bytes 2026-02-22T21:38:09.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2700 IOs, 1174450176 bytes 2026-02-22T21:38:09.222 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2800 IOs, 1175063552 bytes 2026-02-22T21:38:09.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2900 IOs, 1214771200 bytes 2026-02-22T21:38:09.290 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3000 IOs, 1264450560 bytes 2026-02-22T21:38:09.331 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3100 IOs, 1327673344 bytes 2026-02-22T21:38:09.367 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3200 IOs, 1369256960 bytes 2026-02-22T21:38:09.387 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3300 IOs, 1382883328 bytes 2026-02-22T21:38:09.412 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3400 IOs, 1403806720 bytes 2026-02-22T21:38:09.438 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3500 IOs, 1427775488 bytes 2026-02-22T21:38:09.466 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3600 IOs, 1475804160 bytes 2026-02-22T21:38:09.498 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3700 IOs, 1503827968 bytes 2026-02-22T21:38:09.538 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3800 IOs, 1558626304 bytes 2026-02-22T21:38:09.569 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3900 IOs, 1631295488 bytes 2026-02-22T21:38:09.613 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4000 IOs, 1684612096 bytes 2026-02-22T21:38:09.638 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4100 IOs, 1714551808 bytes 2026-02-22T21:38:09.663 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4200 IOs, 1738150912 bytes 2026-02-22T21:38:09.689 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4300 IOs, 1761477632 bytes 2026-02-22T21:38:09.704 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4400 IOs, 1762079744 bytes 2026-02-22T21:38:09.727 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4500 IOs, 1790766080 bytes 2026-02-22T21:38:09.769 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4600 IOs, 1851528192 bytes 2026-02-22T21:38:09.810 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4700 IOs, 1937634304 bytes 2026-02-22T21:38:09.849 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4800 IOs, 2005638144 bytes 2026-02-22T21:38:09.904 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4900 IOs, 2096555008 bytes 2026-02-22T21:38:09.940 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5000 IOs, 2128679936 bytes 2026-02-22T21:38:09.966 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5100 IOs, 2198214656 bytes 2026-02-22T21:38:10.009 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5200 IOs, 2219514880 bytes 2026-02-22T21:38:10.033 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5300 IOs, 2240945152 bytes 2026-02-22T21:38:10.076 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5400 IOs, 2319912960 bytes 2026-02-22T21:38:10.117 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5500 IOs, 2380477440 bytes 2026-02-22T21:38:10.149 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5600 IOs, 2409865216 bytes 2026-02-22T21:38:10.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5700 IOs, 2430940160 bytes 2026-02-22T21:38:10.188 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5800 IOs, 2431559680 bytes 2026-02-22T21:38:10.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5900 IOs, 2480421888 bytes 2026-02-22T21:38:10.247 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6000 IOs, 2504847360 bytes 2026-02-22T21:38:10.269 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6100 IOs, 2522451968 bytes 2026-02-22T21:38:10.313 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6200 IOs, 2585499648 bytes 2026-02-22T21:38:10.345 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6300 IOs, 2648322048 bytes 2026-02-22T21:38:10.390 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6400 IOs, 2720553984 bytes 2026-02-22T21:38:10.423 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6500 IOs, 2732774400 bytes 2026-02-22T21:38:10.452 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6600 IOs, 2763884544 bytes 2026-02-22T21:38:10.481 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6700 IOs, 2810145792 bytes 2026-02-22T21:38:10.511 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6800 IOs, 2849370112 bytes 2026-02-22T21:38:10.557 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6900 IOs, 2914968576 bytes 2026-02-22T21:38:10.610 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7000 IOs, 3002754048 bytes 2026-02-22T21:38:10.644 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7100 IOs, 3047515136 bytes 2026-02-22T21:38:10.690 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7200 IOs, 3121210368 bytes 2026-02-22T21:38:10.744 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7300 IOs, 3220065280 bytes 2026-02-22T21:38:10.772 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7400 IOs, 3242128384 bytes 2026-02-22T21:38:10.793 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7500 IOs, 3273067520 bytes 2026-02-22T21:38:10.817 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7600 IOs, 3288073216 bytes 2026-02-22T21:38:10.850 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7700 IOs, 3334936576 bytes 2026-02-22T21:38:10.876 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7800 IOs, 3393310720 bytes 2026-02-22T21:38:10.924 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7900 IOs, 3436939264 bytes 2026-02-22T21:38:10.953 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8000 IOs, 3471160320 bytes 2026-02-22T21:38:11.001 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8100 IOs, 3553213440 bytes 2026-02-22T21:38:11.031 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8200 IOs, 3592598528 bytes 2026-02-22T21:38:11.049 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8300 IOs, 3601329152 bytes 2026-02-22T21:38:11.063 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8400 IOs, 3601927168 bytes 2026-02-22T21:38:11.099 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8500 IOs, 3658426368 bytes 2026-02-22T21:38:11.132 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8600 IOs, 3698497536 bytes 2026-02-22T21:38:11.158 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8700 IOs, 3711254528 bytes 2026-02-22T21:38:11.183 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8800 IOs, 3735079936 bytes 2026-02-22T21:38:11.203 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8900 IOs, 3752678400 bytes 2026-02-22T21:38:11.231 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9000 IOs, 3776400384 bytes 2026-02-22T21:38:11.254 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9100 IOs, 3804760064 bytes 2026-02-22T21:38:11.280 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9200 IOs, 3834170368 bytes 2026-02-22T21:38:11.324 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9300 IOs, 3908296704 bytes 2026-02-22T21:38:11.349 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9400 IOs, 3933009920 bytes 2026-02-22T21:38:11.370 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9500 IOs, 3937022976 bytes 2026-02-22T21:38:11.410 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9600 IOs, 4003168256 bytes 2026-02-22T21:38:11.443 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9700 IOs, 4047417344 bytes 2026-02-22T21:38:11.482 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9800 IOs, 4128126976 bytes 2026-02-22T21:38:11.514 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9900 IOs, 4156018688 bytes 2026-02-22T21:38:11.541 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10000 IOs, 4185444352 bytes 2026-02-22T21:38:11.560 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10100 IOs, 4195521536 bytes 2026-02-22T21:38:11.586 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10200 IOs, 4236578816 bytes 2026-02-22T21:38:11.635 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10300 IOs, 4297188352 bytes 2026-02-22T21:38:11.678 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10400 IOs, 4360442880 bytes 2026-02-22T21:38:11.704 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10500 IOs, 4389447680 bytes 2026-02-22T21:38:11.755 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10600 IOs, 4480018432 bytes 2026-02-22T21:38:11.789 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10700 IOs, 4511815680 bytes 2026-02-22T21:38:11.827 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10800 IOs, 4564050944 bytes 2026-02-22T21:38:11.862 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10900 IOs, 4612436992 bytes 2026-02-22T21:38:11.891 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11000 IOs, 4639980544 bytes 2026-02-22T21:38:11.918 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11100 IOs, 4667682816 bytes 2026-02-22T21:38:11.951 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11200 IOs, 4709182464 bytes 2026-02-22T21:38:11.993 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11300 IOs, 4777749504 bytes 2026-02-22T21:38:12.017 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11400 IOs, 4790072320 bytes 2026-02-22T21:38:12.052 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11500 IOs, 4839333888 bytes 2026-02-22T21:38:12.090 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11600 IOs, 4917935104 bytes 2026-02-22T21:38:12.121 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11700 IOs, 4947375104 bytes 2026-02-22T21:38:12.155 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11800 IOs, 4970525696 bytes 2026-02-22T21:38:12.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11900 IOs, 5020669952 bytes 2026-02-22T21:38:12.215 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12000 IOs, 5050962944 bytes 2026-02-22T21:38:12.243 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12100 IOs, 5085675520 bytes 2026-02-22T21:38:12.274 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12200 IOs, 5113729024 bytes 2026-02-22T21:38:12.322 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12300 IOs, 5202129920 bytes 2026-02-22T21:38:12.364 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12400 IOs, 5252717568 bytes 2026-02-22T21:38:12.406 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12500 IOs, 5310562304 bytes 2026-02-22T21:38:12.440 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12600 IOs, 5353174016 bytes 2026-02-22T21:38:12.460 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12700 IOs, 5367400448 bytes 2026-02-22T21:38:12.499 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12800 IOs, 5422734336 bytes 2026-02-22T21:38:12.530 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12900 IOs, 5471276032 bytes 2026-02-22T21:38:12.592 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13000 IOs, 5574243328 bytes 2026-02-22T21:38:12.623 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13100 IOs, 5611508736 bytes 2026-02-22T21:38:12.663 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13200 IOs, 5676243968 bytes 2026-02-22T21:38:12.692 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13300 IOs, 5716110336 bytes 2026-02-22T21:38:12.732 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13400 IOs, 5759670272 bytes 2026-02-22T21:38:12.752 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13500 IOs, 5769376768 bytes 2026-02-22T21:38:12.804 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13600 IOs, 5846447104 bytes 2026-02-22T21:38:12.847 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13700 IOs, 5908292608 bytes 2026-02-22T21:38:12.901 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13800 IOs, 5996683264 bytes 2026-02-22T21:38:12.938 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13900 IOs, 6044760064 bytes 2026-02-22T21:38:12.957 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14000 IOs, 6058776576 bytes 2026-02-22T21:38:13.000 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14100 IOs, 6120833024 bytes 2026-02-22T21:38:13.043 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14200 IOs, 6181898240 bytes 2026-02-22T21:38:13.066 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14300 IOs, 6209426432 bytes 2026-02-22T21:38:13.101 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14400 IOs, 6255399936 bytes 2026-02-22T21:38:13.120 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14500 IOs, 6261980160 bytes 2026-02-22T21:38:13.140 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14600 IOs, 6270468096 bytes 2026-02-22T21:38:13.180 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14700 IOs, 6330609664 bytes 2026-02-22T21:38:13.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14800 IOs, 6351144960 bytes 2026-02-22T21:38:13.249 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14900 IOs, 6426628096 bytes 2026-02-22T21:38:13.279 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15000 IOs, 6453693440 bytes 2026-02-22T21:38:13.311 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15100 IOs, 6493120512 bytes 2026-02-22T21:38:13.340 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15200 IOs, 6517432320 bytes 2026-02-22T21:38:13.379 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15300 IOs, 6569433088 bytes 2026-02-22T21:38:13.421 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15400 IOs, 6643616768 bytes 2026-02-22T21:38:13.450 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15500 IOs, 6659428352 bytes 2026-02-22T21:38:13.478 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15600 IOs, 6681673728 bytes 2026-02-22T21:38:13.516 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15700 IOs, 6732957696 bytes 2026-02-22T21:38:13.551 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15800 IOs, 6780966912 bytes 2026-02-22T21:38:13.582 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15900 IOs, 6810095616 bytes 2026-02-22T21:38:13.604 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16000 IOs, 6824648704 bytes 2026-02-22T21:38:13.645 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16100 IOs, 6872658944 bytes 2026-02-22T21:38:13.677 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16200 IOs, 6907062272 bytes 2026-02-22T21:38:13.706 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16300 IOs, 6936133632 bytes 2026-02-22T21:38:13.729 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16400 IOs, 6951716864 bytes 2026-02-22T21:38:13.753 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16500 IOs, 6969668608 bytes 2026-02-22T21:38:13.802 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16600 IOs, 7045439488 bytes 2026-02-22T21:38:13.843 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16700 IOs, 7098470400 bytes 2026-02-22T21:38:13.871 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16800 IOs, 7123909632 bytes 2026-02-22T21:38:13.897 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16900 IOs, 7167535104 bytes 2026-02-22T21:38:13.962 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17000 IOs, 7250966528 bytes 2026-02-22T21:38:13.979 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17100 IOs, 7253690368 bytes 2026-02-22T21:38:14.001 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17200 IOs, 7271242752 bytes 2026-02-22T21:38:14.039 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17300 IOs, 7336907776 bytes 2026-02-22T21:38:14.080 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17400 IOs, 7404267520 bytes 2026-02-22T21:38:14.122 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17500 IOs, 7468384256 bytes 2026-02-22T21:38:14.166 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17600 IOs, 7525396480 bytes 2026-02-22T21:38:14.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17700 IOs, 7572951040 bytes 2026-02-22T21:38:14.243 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17800 IOs, 7636651008 bytes 2026-02-22T21:38:14.280 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17900 IOs, 7667740672 bytes 2026-02-22T21:38:14.312 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18000 IOs, 7709369344 bytes 2026-02-22T21:38:14.342 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ error_code=124 2026-02-22T21:38:14.342 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set -e 2026-02-22T21:38:14.343 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' 124 -eq 124 ']' 2026-02-22T21:38:14.343 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:38:14.343 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T21:38:14.343 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:38:14.343 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:38:14.343 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:38:14.343 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state_pattern=up+replaying 2026-02-22T21:38:14.343 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local description_pattern= 2026-02-22T21:38:14.343 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local service_pattern= 2026-02-22T21:38:14.343 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:38:14.343 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T21:38:15.342 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T21:38:15.342 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:38:15.342 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:38:15.342 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:38:15.342 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state_pattern=up+replaying 2026-02-22T21:38:15.342 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local description_pattern= 2026-02-22T21:38:15.342 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local service_pattern= 2026-02-22T21:38:15.342 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status 2026-02-22T21:38:15.343 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ CEPH_ARGS= 2026-02-22T21:38:15.343 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:38:15.374 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ status='test: 2026-02-22T21:38:15.374 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: global_id: b410963b-cae9-4337-a6ef-da65321c9374 2026-02-22T21:38:15.375 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: state: up+replaying 2026-02-22T21:38:15.375 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771796263,"replay_state":"idle"} 2026-02-22T21:38:15.375 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: service: mirror.0 on trial026 2026-02-22T21:38:15.375 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: last_update: 2026-02-22 21:38:05 2026-02-22T21:38:15.375 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: peer_sites: 2026-02-22T21:38:15.375 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: name: cluster2 2026-02-22T21:38:15.375 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: state: up+stopped 2026-02-22T21:38:15.375 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: description: local image is primary 2026-02-22T21:38:15.375 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: last_update: 2026-02-22 21:38:05' 2026-02-22T21:38:15.375 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T21:38:15.375 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: state: up+replaying 2026-02-22T21:38:15.375 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep '^ description: .*' 2026-02-22T21:38:15.376 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771796263,"replay_state":"idle"} 2026-02-22T21:38:15.376 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n '' ']' 2026-02-22T21:38:15.376 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ echo up+replaying 2026-02-22T21:38:15.376 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep '^up+' 2026-02-22T21:38:15.377 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:up+replaying 2026-02-22T21:38:15.377 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:38:15.378 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: service: mirror.0 on trial026 2026-02-22T21:38:15.378 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local last_update 2026-02-22T21:38:15.379 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:38:15.380 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ last_update='2026-02-22 21:38:05' 2026-02-22T21:38:15.380 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 21:38:05' 2026-02-22T21:38:15.380 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:38:15.380 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:38:15.380 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:38:15.380 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state_pattern=up+replaying 2026-02-22T21:38:15.381 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local 'prev_last_update=2026-02-22 21:38:05' 2026-02-22T21:38:15.381 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status 2026-02-22T21:38:15.381 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ CEPH_ARGS= 2026-02-22T21:38:15.381 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:38:15.413 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ status='OKOKOK142074258mirror.0trial02619.2.3-689-gf90f4eaetrueOK42104261mirror.1trial02619.2.3-689-gf90f4eaefalseOK42144294mirror.2trial02619.2.3-689-gf90f4eaefalseOK42184265mirror.3trial02619.2.3-689-gf90f4eaefalseOKtestb410963b-cae9-4337-a6ef-da65321c9374up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771796263,"replay_state":"idle"}42074258mirror.0trial0262026-02-22 21:38:05cluster2cfa81703-817a-4074-adbc-5e6e200790bfup+stoppedlocal image is primary2026-02-22 21:38:05' 2026-02-22T21:38:15.413 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local last_update state 2026-02-22T21:38:15.413 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:38:15.415 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ last_update='2026-02-22 21:38:05' 2026-02-22T21:38:15.416 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:38:15.417 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ state=up+replaying 2026-02-22T21:38:15.417 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ echo up+replaying 2026-02-22T21:38:15.417 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep up+replaying 2026-02-22T21:38:15.418 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:up+replaying 2026-02-22T21:38:15.418 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:38:15.418 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:38:15.418 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ snap_name=snap1 2026-02-22T21:38:15.419 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_snap cluster2 mirror test snap1 2026-02-22T21:38:15.419 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:38:15.419 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:38:15.419 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:38:15.419 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap1 2026-02-22T21:38:15.419 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap1 --debug-rbd=20 --debug-journaler=20 2026-02-22T21:38:39.327 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.43538221378452546) > weight (0.25) 2026-02-22T21:38:39.327 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.3814445784124476) > weight (0.25) 2026-02-22T21:38:39.327 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.5744018000716425) > weight (0.25) 2026-02-22T21:38:39.327 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.3 with skip (0.45871321425059586) > weight (0.25) 2026-02-22T21:38:39.327 INFO:tasks.rbd_mirror_thrash:waiting for 102.35159378075423 secs before thrashing 2026-02-22T21:38:45.103 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T21:38:45.103 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:38:45.104 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:38:45.104 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:38:45.104 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T21:38:45.104 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:38:45.104 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:38:45.104 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:38:45.104 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=started 2026-02-22T21:38:45.104 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T21:38:45.104 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:38:45.104 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T21:38:46.104 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T21:38:46.104 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:38:46.104 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:38:46.104 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:38:46.104 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=started 2026-02-22T21:38:46.104 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_result 2026-02-22T21:38:46.104 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=stopped 2026-02-22T21:38:46.105 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:38:46.105 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ grep -i state 2026-02-22T21:38:46.105 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_instance=cluster1 2026-02-22T21:38:46.105 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:38:46.105 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T21:38:46.105 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance=cluster1 2026-02-22T21:38:46.105 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local loop_instance 2026-02-22T21:38:46.105 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:38:46.105 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sleep 0 2026-02-22T21:38:46.106 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:38:46.106 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ seq 0 3 2026-02-22T21:38:46.108 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:38:46.108 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:38:46.108 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:0 2026-02-22T21:38:46.108 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:38:46.108 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:38:46.108 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:38:46.108 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:0 2026-02-22T21:38:46.108 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:38:46.108 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:38:46.108 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:38:46.108 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=0 2026-02-22T21:38:46.108 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:38:46.108 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:38:46.108 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:38:46.108 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=0 2026-02-22T21:38:46.108 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T21:38:46.108 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:38:46.108 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:38:46.108 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:38:46.108 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:38:46.109 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:38:46.109 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:0 2026-02-22T21:38:46.109 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:38:46.109 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:38:46.109 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:38:46.109 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T21:38:46.109 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:38:46.109 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:38:46.109 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:38:46.109 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=0 2026-02-22T21:38:46.109 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=0 2026-02-22T21:38:46.109 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:38:46.121 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:38:46.121 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:38:46.121 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:38:46.122 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:38:46.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ return 0 2026-02-22T21:38:46.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:38:46.169 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:38:46.169 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep -i Replaying 2026-02-22T21:38:46.169 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: "state": "Replaying" 2026-02-22T21:38:46.171 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_state=started 2026-02-22T21:38:46.171 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test started = started 2026-02-22T21:38:46.171 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:38:46.171 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2026-02-22T21:38:46.171 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T21:38:46.171 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:38:46.171 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:38:46.171 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:38:46.171 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = journal ']' 2026-02-22T21:38:46.171 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T21:38:46.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-22T21:38:46.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:38:46.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:38:46.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:38:46.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-22T21:38:47.103 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Snapshot ID: 5 2026-02-22T21:38:47.109 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test 2026-02-22T21:38:47.109 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T21:38:47.110 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:38:47.110 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:38:47.110 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:38:47.110 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster2-mirror-test.status 2026-02-22T21:38:47.110 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster2-mirror-test.status 2026-02-22T21:38:47.110 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T21:38:47.111 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_log=/tmp/tmp.j44LZPowal/cluster2-mirror-test.status 2026-02-22T21:38:47.111 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster1-mirror-test.status 2026-02-22T21:38:47.111 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster1-mirror-test.status 2026-02-22T21:38:47.111 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T21:38:47.112 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_status_log=/tmp/tmp.j44LZPowal/cluster1-mirror-test.status 2026-02-22T21:38:47.113 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.j44LZPowal/cluster2-mirror-test.status 2026-02-22T21:38:47.113 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:38:47.113 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:38:47.113 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:38:47.113 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster2-mirror-test.status 2026-02-22T21:38:47.113 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-22T21:38:47.113 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:38:47.140 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T21:38:47.142 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snapshot_id=5 2026-02-22T21:38:47.142 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:38:47.142 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:38:47.142 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.2 2026-02-22T21:38:47.343 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.j44LZPowal/cluster1-mirror-test.status 2026-02-22T21:38:47.343 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:38:47.344 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:38:47.344 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:38:47.344 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-test.status 2026-02-22T21:38:47.344 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T21:38:47.344 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:38:47.374 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:38:47.375 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id=3 2026-02-22T21:38:47.375 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 5 = 3 2026-02-22T21:38:47.375 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:38:47.376 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.4 2026-02-22T21:38:47.777 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.j44LZPowal/cluster1-mirror-test.status 2026-02-22T21:38:47.777 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:38:47.777 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:38:47.778 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:38:47.778 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-test.status 2026-02-22T21:38:47.778 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T21:38:47.778 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:38:47.806 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:38:47.806 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:38:47.808 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T21:38:47.808 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T21:38:47.809 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T21:38:47.809 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T21:38:47.809 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 5 = '' 2026-02-22T21:38:47.809 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:38:47.809 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.8 2026-02-22T21:38:48.609 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.j44LZPowal/cluster1-mirror-test.status 2026-02-22T21:38:48.609 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:38:48.610 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:38:48.610 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:38:48.610 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-test.status 2026-02-22T21:38:48.610 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T21:38:48.610 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:38:48.648 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:38:48.649 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:38:48.650 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T21:38:48.650 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T21:38:48.650 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T21:38:48.651 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T21:38:48.651 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 5 = '' 2026-02-22T21:38:48.651 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:38:48.651 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1.6 2026-02-22T21:38:50.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.j44LZPowal/cluster1-mirror-test.status 2026-02-22T21:38:50.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:38:50.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:38:50.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:38:50.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-test.status 2026-02-22T21:38:50.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T21:38:50.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:38:50.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:38:50.286 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id=5 2026-02-22T21:38:50.286 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 5 = 5 2026-02-22T21:38:50.287 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:38:50.287 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snap_present cluster1 mirror test snap1 2026-02-22T21:38:50.287 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:38:50.287 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:38:50.287 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:38:50.287 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap1 2026-02-22T21:38:50.287 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T21:38:50.287 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T21:38:50.287 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T21:38:51.287 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 info mirror/test@snap1 2026-02-22T21:38:51.312 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'test': 2026-02-22T21:38:51.313 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 512 MiB in 128 objects 2026-02-22T21:38:51.313 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T21:38:51.313 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 2 2026-02-22T21:38:51.313 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 10a25aa14179 2026-02-22T21:38:51.313 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.10a25aa14179 2026-02-22T21:38:51.313 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T21:38:51.313 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock, non-primary 2026-02-22T21:38:51.313 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T21:38:51.313 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T21:38:51.314 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:38:05 2026 2026-02-22T21:38:51.314 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:38:05 2026 2026-02-22T21:38:51.314 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:38:05 2026 2026-02-22T21:38:51.314 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: protected: False 2026-02-22T21:38:51.314 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T21:38:51.314 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T21:38:51.314 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: b410963b-cae9-4337-a6ef-da65321c9374 2026-02-22T21:38:51.314 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: false 2026-02-22T21:38:51.315 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:38:51.315 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n '' ']' 2026-02-22T21:38:51.316 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ compare_image_snaps mirror test snap1 2026-02-22T21:38:51.316 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:38:51.316 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:38:51.316 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap1 2026-02-22T21:38:51.316 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local ret=0 2026-02-22T21:38:51.316 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local rmt_export=/tmp/tmp.j44LZPowal/cluster2-mirror-test.export 2026-02-22T21:38:51.316 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local loc_export=/tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:38:51.316 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:38:51.316 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror export test@snap1 /tmp/tmp.j44LZPowal/cluster2-mirror-test.export 2026-02-22T21:38:51.648 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-22T21:38:51.658 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 -p mirror export test@snap1 /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:38:52.057 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-22T21:38:52.070 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ cmp /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:38:52.246 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:38:52.371 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:38:52.371 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ clean_snap_name=snap1-clean 2026-02-22T21:38:52.371 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_snap cluster2 mirror test snap1-clean 2026-02-22T21:38:52.372 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:38:52.372 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:38:52.372 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:38:52.372 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap1-clean 2026-02-22T21:38:52.372 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap1-clean --debug-rbd=20 --debug-journaler=20 2026-02-22T21:38:52.459 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 10` 2026-02-22T21:38:52.459 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ stress_write_image cluster2 mirror test 2026-02-22T21:38:52.459 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:38:52.459 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:38:52.459 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:38:52.459 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-22T21:38:52.460 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local duration=14 2026-02-22T21:38:52.460 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set +e 2026-02-22T21:38:52.460 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ timeout 14s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2026-02-22T21:38:52.581 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:100 IOs, 85199872 bytes 2026-02-22T21:38:52.625 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:200 IOs, 124220416 bytes 2026-02-22T21:38:52.646 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:300 IOs, 138054656 bytes 2026-02-22T21:38:52.674 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:400 IOs, 155072512 bytes 2026-02-22T21:38:52.699 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:500 IOs, 173223936 bytes 2026-02-22T21:38:52.724 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:600 IOs, 199821312 bytes 2026-02-22T21:38:52.767 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:700 IOs, 275884032 bytes 2026-02-22T21:38:52.798 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:800 IOs, 297119744 bytes 2026-02-22T21:38:52.831 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:900 IOs, 340194304 bytes 2026-02-22T21:38:52.868 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1000 IOs, 399632384 bytes 2026-02-22T21:38:52.897 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1100 IOs, 443877376 bytes 2026-02-22T21:38:52.914 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1200 IOs, 451154944 bytes 2026-02-22T21:38:52.940 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1300 IOs, 493811712 bytes 2026-02-22T21:38:52.969 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1400 IOs, 524417024 bytes 2026-02-22T21:38:53.016 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1500 IOs, 593525760 bytes 2026-02-22T21:38:53.044 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1600 IOs, 622718976 bytes 2026-02-22T21:38:53.089 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1700 IOs, 696368128 bytes 2026-02-22T21:38:53.111 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1800 IOs, 715497472 bytes 2026-02-22T21:38:53.128 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1900 IOs, 716072960 bytes 2026-02-22T21:38:53.149 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2000 IOs, 737816576 bytes 2026-02-22T21:38:53.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2100 IOs, 738416640 bytes 2026-02-22T21:38:53.218 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2200 IOs, 821861376 bytes 2026-02-22T21:38:53.252 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2300 IOs, 866577408 bytes 2026-02-22T21:38:53.276 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2400 IOs, 891006976 bytes 2026-02-22T21:38:53.308 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2500 IOs, 926559232 bytes 2026-02-22T21:38:53.336 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2600 IOs, 954100736 bytes 2026-02-22T21:38:53.376 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2700 IOs, 1002416128 bytes 2026-02-22T21:38:53.394 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2800 IOs, 1028189184 bytes 2026-02-22T21:38:53.424 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2900 IOs, 1059220480 bytes 2026-02-22T21:38:53.446 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3000 IOs, 1068583936 bytes 2026-02-22T21:38:53.473 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3100 IOs, 1089483776 bytes 2026-02-22T21:38:53.493 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3200 IOs, 1116890112 bytes 2026-02-22T21:38:53.529 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3300 IOs, 1141331968 bytes 2026-02-22T21:38:53.554 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3400 IOs, 1159615488 bytes 2026-02-22T21:38:53.602 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3500 IOs, 1235756032 bytes 2026-02-22T21:38:53.629 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3600 IOs, 1269945344 bytes 2026-02-22T21:38:53.660 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3700 IOs, 1296444416 bytes 2026-02-22T21:38:53.683 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3800 IOs, 1316055040 bytes 2026-02-22T21:38:53.729 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3900 IOs, 1376092160 bytes 2026-02-22T21:38:53.767 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4000 IOs, 1421060096 bytes 2026-02-22T21:38:53.793 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4100 IOs, 1447900160 bytes 2026-02-22T21:38:53.824 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4200 IOs, 1484640256 bytes 2026-02-22T21:38:53.870 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4300 IOs, 1549701120 bytes 2026-02-22T21:38:53.901 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4400 IOs, 1587523584 bytes 2026-02-22T21:38:53.920 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4500 IOs, 1592841216 bytes 2026-02-22T21:38:53.972 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4600 IOs, 1672512512 bytes 2026-02-22T21:38:53.998 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4700 IOs, 1695906816 bytes 2026-02-22T21:38:54.016 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4800 IOs, 1696529408 bytes 2026-02-22T21:38:54.054 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4900 IOs, 1738531840 bytes 2026-02-22T21:38:54.089 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5000 IOs, 1776096256 bytes 2026-02-22T21:38:54.138 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5100 IOs, 1851206656 bytes 2026-02-22T21:38:54.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5200 IOs, 1955873792 bytes 2026-02-22T21:38:54.230 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5300 IOs, 2017748992 bytes 2026-02-22T21:38:54.257 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5400 IOs, 2029526016 bytes 2026-02-22T21:38:54.299 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5500 IOs, 2091355136 bytes 2026-02-22T21:38:54.322 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.0 2026-02-22T21:38:54.323 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Sent signal 15 2026-02-22T21:38:54.323 INFO:tasks.rbd_mirror_thrash:waiting for 1.8799763752222154 secs before reviving daemons 2026-02-22T21:38:54.324 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial026.stderr:2026-02-22T21:38:54.322+0000 7fbff1443640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 (PID: 48105) UID: 1000 2026-02-22T21:38:54.331 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5600 IOs, 2128588800 bytes 2026-02-22T21:38:54.359 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5700 IOs, 2146098176 bytes 2026-02-22T21:38:54.398 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5800 IOs, 2199678976 bytes 2026-02-22T21:38:54.421 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5900 IOs, 2220502016 bytes 2026-02-22T21:38:54.463 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6000 IOs, 2284285952 bytes 2026-02-22T21:38:54.494 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6100 IOs, 2322680832 bytes 2026-02-22T21:38:54.540 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6200 IOs, 2370150400 bytes 2026-02-22T21:38:54.608 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6300 IOs, 2496133120 bytes 2026-02-22T21:38:54.642 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6400 IOs, 2522281984 bytes 2026-02-22T21:38:54.663 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6500 IOs, 2529412096 bytes 2026-02-22T21:38:54.705 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6600 IOs, 2598324224 bytes 2026-02-22T21:38:54.739 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6700 IOs, 2644149248 bytes 2026-02-22T21:38:54.772 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6800 IOs, 2682460160 bytes 2026-02-22T21:38:54.794 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6900 IOs, 2695880704 bytes 2026-02-22T21:38:54.822 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7000 IOs, 2723988480 bytes 2026-02-22T21:38:54.845 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7100 IOs, 2741255168 bytes 2026-02-22T21:38:54.890 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7200 IOs, 2809365504 bytes 2026-02-22T21:38:54.924 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7300 IOs, 2855067648 bytes 2026-02-22T21:38:54.958 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7400 IOs, 2896964608 bytes 2026-02-22T21:38:54.986 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7500 IOs, 2929983488 bytes 2026-02-22T21:38:55.038 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7600 IOs, 3014279168 bytes 2026-02-22T21:38:55.073 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7700 IOs, 3059831808 bytes 2026-02-22T21:38:55.113 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7800 IOs, 3120198656 bytes 2026-02-22T21:38:55.135 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7900 IOs, 3125990400 bytes 2026-02-22T21:38:55.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8000 IOs, 3232218112 bytes 2026-02-22T21:38:55.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8100 IOs, 3250240512 bytes 2026-02-22T21:38:55.249 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8200 IOs, 3275976704 bytes 2026-02-22T21:38:55.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8300 IOs, 3295218688 bytes 2026-02-22T21:38:55.312 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8400 IOs, 3356903424 bytes 2026-02-22T21:38:55.342 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8500 IOs, 3382343680 bytes 2026-02-22T21:38:55.385 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8600 IOs, 3445658624 bytes 2026-02-22T21:38:55.470 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8700 IOs, 3598946304 bytes 2026-02-22T21:38:55.504 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8800 IOs, 3637244928 bytes 2026-02-22T21:38:55.535 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8900 IOs, 3677168640 bytes 2026-02-22T21:38:55.578 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9000 IOs, 3728745472 bytes 2026-02-22T21:38:55.603 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9100 IOs, 3745696768 bytes 2026-02-22T21:38:55.642 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9200 IOs, 3790062592 bytes 2026-02-22T21:38:55.678 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9300 IOs, 3845139456 bytes 2026-02-22T21:38:55.727 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9400 IOs, 3907869696 bytes 2026-02-22T21:38:55.765 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9500 IOs, 3954216960 bytes 2026-02-22T21:38:55.792 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9600 IOs, 3975023616 bytes 2026-02-22T21:38:55.833 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9700 IOs, 4032356352 bytes 2026-02-22T21:38:55.860 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9800 IOs, 4048843776 bytes 2026-02-22T21:38:55.887 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9900 IOs, 4066180096 bytes 2026-02-22T21:38:55.906 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10000 IOs, 4081405952 bytes 2026-02-22T21:38:55.940 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10100 IOs, 4128731136 bytes 2026-02-22T21:38:55.989 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10200 IOs, 4198436864 bytes 2026-02-22T21:38:56.045 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10300 IOs, 4275715072 bytes 2026-02-22T21:38:56.074 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10400 IOs, 4309324800 bytes 2026-02-22T21:38:56.163 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10500 IOs, 4444893184 bytes 2026-02-22T21:38:56.204 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.0 2026-02-22T21:38:56.204 INFO:teuthology.orchestra.run:waiting for 600 2026-02-22T21:38:56.204 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.0 2026-02-22T21:38:56.204 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Restarting daemon 2026-02-22T21:38:56.205 DEBUG:teuthology.orchestra.run.trial026:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 2026-02-22T21:38:56.207 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Started 2026-02-22T21:38:56.207 INFO:tasks.rbd_mirror_thrash:waiting for 83.01393053662201 secs before thrashing 2026-02-22T21:38:56.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10600 IOs, 4514002944 bytes 2026-02-22T21:38:56.244 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10700 IOs, 4546883584 bytes 2026-02-22T21:38:56.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10800 IOs, 4579511296 bytes 2026-02-22T21:38:56.314 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10900 IOs, 4617566208 bytes 2026-02-22T21:38:56.344 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11000 IOs, 4670938112 bytes 2026-02-22T21:38:56.394 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11100 IOs, 4724343808 bytes 2026-02-22T21:38:56.451 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11200 IOs, 4806148096 bytes 2026-02-22T21:38:56.478 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11300 IOs, 4820193280 bytes 2026-02-22T21:38:56.515 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11400 IOs, 4866121728 bytes 2026-02-22T21:38:56.544 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11500 IOs, 4893986816 bytes 2026-02-22T21:38:56.561 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11600 IOs, 4894795776 bytes 2026-02-22T21:38:56.591 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11700 IOs, 4928932864 bytes 2026-02-22T21:38:56.621 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11800 IOs, 4984046592 bytes 2026-02-22T21:38:56.659 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11900 IOs, 5007679488 bytes 2026-02-22T21:38:56.686 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12000 IOs, 5028292608 bytes 2026-02-22T21:38:56.715 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12100 IOs, 5056149504 bytes 2026-02-22T21:38:56.750 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12200 IOs, 5096221696 bytes 2026-02-22T21:38:56.794 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12300 IOs, 5163931648 bytes 2026-02-22T21:38:56.825 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12400 IOs, 5191742464 bytes 2026-02-22T21:38:56.852 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12500 IOs, 5216945152 bytes 2026-02-22T21:38:56.885 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12600 IOs, 5258900480 bytes 2026-02-22T21:38:56.911 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12700 IOs, 5278969856 bytes 2026-02-22T21:38:56.947 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12800 IOs, 5319053312 bytes 2026-02-22T21:38:56.975 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12900 IOs, 5336379392 bytes 2026-02-22T21:38:57.021 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13000 IOs, 5408374784 bytes 2026-02-22T21:38:57.055 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13100 IOs, 5448043520 bytes 2026-02-22T21:38:57.085 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13200 IOs, 5478126592 bytes 2026-02-22T21:38:57.111 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13300 IOs, 5516693504 bytes 2026-02-22T21:38:57.163 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13400 IOs, 5582310400 bytes 2026-02-22T21:38:57.187 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13500 IOs, 5604676608 bytes 2026-02-22T21:38:57.215 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13600 IOs, 5620524032 bytes 2026-02-22T21:38:57.264 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13700 IOs, 5692250112 bytes 2026-02-22T21:38:57.303 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13800 IOs, 5734206464 bytes 2026-02-22T21:38:57.320 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13900 IOs, 5734784000 bytes 2026-02-22T21:38:57.344 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14000 IOs, 5752810496 bytes 2026-02-22T21:38:57.379 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14100 IOs, 5812685824 bytes 2026-02-22T21:38:57.430 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14200 IOs, 5876357120 bytes 2026-02-22T21:38:57.458 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14300 IOs, 5902800896 bytes 2026-02-22T21:38:57.482 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14400 IOs, 5923251200 bytes 2026-02-22T21:38:57.527 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14500 IOs, 5986082816 bytes 2026-02-22T21:38:57.588 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14600 IOs, 6078279680 bytes 2026-02-22T21:38:57.607 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14700 IOs, 6084021248 bytes 2026-02-22T21:38:57.636 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14800 IOs, 6114878464 bytes 2026-02-22T21:38:57.654 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14900 IOs, 6129270784 bytes 2026-02-22T21:38:57.682 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15000 IOs, 6158738432 bytes 2026-02-22T21:38:57.727 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15100 IOs, 6218762240 bytes 2026-02-22T21:38:57.766 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15200 IOs, 6261117952 bytes 2026-02-22T21:38:57.796 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15300 IOs, 6298425344 bytes 2026-02-22T21:38:57.839 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15400 IOs, 6350171136 bytes 2026-02-22T21:38:57.870 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15500 IOs, 6387137536 bytes 2026-02-22T21:38:57.923 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15600 IOs, 6478851072 bytes 2026-02-22T21:38:57.955 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15700 IOs, 6522237952 bytes 2026-02-22T21:38:57.995 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15800 IOs, 6580795392 bytes 2026-02-22T21:38:58.037 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15900 IOs, 6634284032 bytes 2026-02-22T21:38:58.069 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16000 IOs, 6660051968 bytes 2026-02-22T21:38:58.120 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16100 IOs, 6744155136 bytes 2026-02-22T21:38:58.151 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16200 IOs, 6781782016 bytes 2026-02-22T21:38:58.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16300 IOs, 6812301312 bytes 2026-02-22T21:38:58.227 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16400 IOs, 6854899712 bytes 2026-02-22T21:38:58.250 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16500 IOs, 6858168320 bytes 2026-02-22T21:38:58.270 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16600 IOs, 6864331776 bytes 2026-02-22T21:38:58.303 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16700 IOs, 6899738624 bytes 2026-02-22T21:38:58.332 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16800 IOs, 6933216256 bytes 2026-02-22T21:38:58.362 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16900 IOs, 6961394688 bytes 2026-02-22T21:38:58.408 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17000 IOs, 7033511936 bytes 2026-02-22T21:38:58.430 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17100 IOs, 7040278528 bytes 2026-02-22T21:38:58.458 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17200 IOs, 7060936704 bytes 2026-02-22T21:38:58.496 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17300 IOs, 7101775872 bytes 2026-02-22T21:38:58.539 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17400 IOs, 7155050496 bytes 2026-02-22T21:38:58.557 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17500 IOs, 7155668992 bytes 2026-02-22T21:38:58.583 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17600 IOs, 7195890688 bytes 2026-02-22T21:38:58.613 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17700 IOs, 7199919104 bytes 2026-02-22T21:38:58.642 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17800 IOs, 7224735744 bytes 2026-02-22T21:38:58.693 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17900 IOs, 7316527104 bytes 2026-02-22T21:38:58.726 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18000 IOs, 7339000832 bytes 2026-02-22T21:38:58.750 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18100 IOs, 7373034496 bytes 2026-02-22T21:38:58.775 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18200 IOs, 7373643776 bytes 2026-02-22T21:38:58.818 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18300 IOs, 7437403136 bytes 2026-02-22T21:38:58.871 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18400 IOs, 7516998656 bytes 2026-02-22T21:38:58.903 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18500 IOs, 7545102336 bytes 2026-02-22T21:38:58.945 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18600 IOs, 7606038528 bytes 2026-02-22T21:38:58.998 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18700 IOs, 7690881024 bytes 2026-02-22T21:38:59.038 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18800 IOs, 7728721920 bytes 2026-02-22T21:38:59.070 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18900 IOs, 7752435712 bytes 2026-02-22T21:38:59.101 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19000 IOs, 7788191744 bytes 2026-02-22T21:38:59.131 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19100 IOs, 7822699520 bytes 2026-02-22T21:38:59.162 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19200 IOs, 7844561920 bytes 2026-02-22T21:38:59.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19300 IOs, 7915346944 bytes 2026-02-22T21:38:59.229 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19400 IOs, 7920344064 bytes 2026-02-22T21:38:59.285 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19500 IOs, 8010267648 bytes 2026-02-22T21:38:59.312 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19600 IOs, 8032792576 bytes 2026-02-22T21:38:59.333 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19700 IOs, 8044374016 bytes 2026-02-22T21:38:59.379 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19800 IOs, 8105720832 bytes 2026-02-22T21:38:59.419 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19900 IOs, 8161665024 bytes 2026-02-22T21:38:59.442 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20000 IOs, 8196282368 bytes 2026-02-22T21:38:59.482 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20100 IOs, 8244075520 bytes 2026-02-22T21:38:59.525 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20200 IOs, 8279327744 bytes 2026-02-22T21:38:59.564 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20300 IOs, 8328281088 bytes 2026-02-22T21:38:59.609 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20400 IOs, 8391325696 bytes 2026-02-22T21:38:59.638 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20500 IOs, 8413931520 bytes 2026-02-22T21:38:59.670 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20600 IOs, 8453168128 bytes 2026-02-22T21:38:59.700 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20700 IOs, 8483472384 bytes 2026-02-22T21:38:59.729 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20800 IOs, 8507454464 bytes 2026-02-22T21:38:59.782 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20900 IOs, 8592859136 bytes 2026-02-22T21:38:59.825 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21000 IOs, 8638567424 bytes 2026-02-22T21:38:59.865 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21100 IOs, 8693151744 bytes 2026-02-22T21:38:59.900 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21200 IOs, 8734260224 bytes 2026-02-22T21:38:59.925 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21300 IOs, 8751635456 bytes 2026-02-22T21:38:59.948 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21400 IOs, 8764284928 bytes 2026-02-22T21:38:59.973 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21500 IOs, 8790855680 bytes 2026-02-22T21:39:00.007 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21600 IOs, 8810129408 bytes 2026-02-22T21:39:00.057 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21700 IOs, 8890735616 bytes 2026-02-22T21:39:00.108 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21800 IOs, 8978903040 bytes 2026-02-22T21:39:00.128 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21900 IOs, 8989456384 bytes 2026-02-22T21:39:00.149 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22000 IOs, 8999731200 bytes 2026-02-22T21:39:00.196 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22100 IOs, 9073790976 bytes 2026-02-22T21:39:00.228 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22200 IOs, 9099654144 bytes 2026-02-22T21:39:00.254 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22300 IOs, 9104989184 bytes 2026-02-22T21:39:00.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22400 IOs, 9154577408 bytes 2026-02-22T21:39:00.326 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22500 IOs, 9183934464 bytes 2026-02-22T21:39:00.356 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22600 IOs, 9209357312 bytes 2026-02-22T21:39:00.384 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22700 IOs, 9232381952 bytes 2026-02-22T21:39:00.432 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22800 IOs, 9308274688 bytes 2026-02-22T21:39:00.474 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22900 IOs, 9364218880 bytes 2026-02-22T21:39:00.509 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23000 IOs, 9408034816 bytes 2026-02-22T21:39:00.539 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23100 IOs, 9430104064 bytes 2026-02-22T21:39:00.585 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23200 IOs, 9504006144 bytes 2026-02-22T21:39:00.603 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23300 IOs, 9504628736 bytes 2026-02-22T21:39:00.673 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23400 IOs, 9619816448 bytes 2026-02-22T21:39:00.699 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23500 IOs, 9637606400 bytes 2026-02-22T21:39:00.726 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23600 IOs, 9654523904 bytes 2026-02-22T21:39:00.760 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23700 IOs, 9683813376 bytes 2026-02-22T21:39:00.799 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23800 IOs, 9720523776 bytes 2026-02-22T21:39:00.834 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23900 IOs, 9760627712 bytes 2026-02-22T21:39:00.883 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24000 IOs, 9840134144 bytes 2026-02-22T21:39:00.913 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24100 IOs, 9863604224 bytes 2026-02-22T21:39:00.948 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24200 IOs, 9895785472 bytes 2026-02-22T21:39:00.976 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24300 IOs, 9932813312 bytes 2026-02-22T21:39:01.009 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24400 IOs, 9950889984 bytes 2026-02-22T21:39:01.079 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24500 IOs, 10060204032 bytes 2026-02-22T21:39:01.115 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24600 IOs, 10108965888 bytes 2026-02-22T21:39:01.139 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24700 IOs, 10114269184 bytes 2026-02-22T21:39:01.159 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24800 IOs, 10118028288 bytes 2026-02-22T21:39:01.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24900 IOs, 10157702144 bytes 2026-02-22T21:39:01.237 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25000 IOs, 10217114624 bytes 2026-02-22T21:39:01.280 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25100 IOs, 10254147584 bytes 2026-02-22T21:39:01.307 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25200 IOs, 10275995648 bytes 2026-02-22T21:39:01.336 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25300 IOs, 10296172544 bytes 2026-02-22T21:39:01.357 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25400 IOs, 10304592896 bytes 2026-02-22T21:39:01.404 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25500 IOs, 10365965312 bytes 2026-02-22T21:39:01.434 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25600 IOs, 10393422848 bytes 2026-02-22T21:39:01.468 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25700 IOs, 10426743808 bytes 2026-02-22T21:39:01.520 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25800 IOs, 10496528384 bytes 2026-02-22T21:39:01.549 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25900 IOs, 10521067520 bytes 2026-02-22T21:39:01.567 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:26000 IOs, 10521686016 bytes 2026-02-22T21:39:01.587 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:26100 IOs, 10530166784 bytes 2026-02-22T21:39:01.605 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:26200 IOs, 10544916480 bytes 2026-02-22T21:39:01.636 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:26300 IOs, 10565500928 bytes 2026-02-22T21:39:01.668 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:26400 IOs, 10603437056 bytes 2026-02-22T21:39:01.708 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:26500 IOs, 10656864256 bytes 2026-02-22T21:39:01.740 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:26600 IOs, 10688924672 bytes 2026-02-22T21:39:01.767 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:26700 IOs, 10711616512 bytes 2026-02-22T21:39:01.794 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:26800 IOs, 10733739008 bytes 2026-02-22T21:39:01.831 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:26900 IOs, 10795055104 bytes 2026-02-22T21:39:01.880 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:27000 IOs, 10841766912 bytes 2026-02-22T21:39:01.910 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:27100 IOs, 10868004864 bytes 2026-02-22T21:39:01.949 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:27200 IOs, 10908277760 bytes 2026-02-22T21:39:01.984 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:27300 IOs, 10940309504 bytes 2026-02-22T21:39:02.026 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:27400 IOs, 10982451200 bytes 2026-02-22T21:39:02.072 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:27500 IOs, 11045733376 bytes 2026-02-22T21:39:02.127 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:27600 IOs, 11142845440 bytes 2026-02-22T21:39:02.154 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:27700 IOs, 11146872832 bytes 2026-02-22T21:39:02.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:27800 IOs, 11182947328 bytes 2026-02-22T21:39:02.227 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:27900 IOs, 11237926912 bytes 2026-02-22T21:39:02.292 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:28000 IOs, 11345826816 bytes 2026-02-22T21:39:02.346 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:28100 IOs, 11423305728 bytes 2026-02-22T21:39:02.379 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:28200 IOs, 11442459648 bytes 2026-02-22T21:39:02.405 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:28300 IOs, 11467011072 bytes 2026-02-22T21:39:02.440 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:28400 IOs, 11495078912 bytes 2026-02-22T21:39:02.488 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:28500 IOs, 11559727104 bytes 2026-02-22T21:39:02.530 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:28600 IOs, 11613057024 bytes 2026-02-22T21:39:02.556 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:28700 IOs, 11636989952 bytes 2026-02-22T21:39:02.609 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:28800 IOs, 11728941056 bytes 2026-02-22T21:39:02.662 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:28900 IOs, 11815571456 bytes 2026-02-22T21:39:02.694 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:29000 IOs, 11850262528 bytes 2026-02-22T21:39:02.730 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:29100 IOs, 11890957312 bytes 2026-02-22T21:39:02.748 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:29200 IOs, 11891580928 bytes 2026-02-22T21:39:02.798 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:29300 IOs, 11967351808 bytes 2026-02-22T21:39:02.820 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:29400 IOs, 11967964160 bytes 2026-02-22T21:39:02.867 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:29500 IOs, 12035564544 bytes 2026-02-22T21:39:02.898 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:29600 IOs, 12077022208 bytes 2026-02-22T21:39:02.944 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:29700 IOs, 12132629504 bytes 2026-02-22T21:39:02.993 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:29800 IOs, 12198210560 bytes 2026-02-22T21:39:03.046 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:29900 IOs, 12276656128 bytes 2026-02-22T21:39:03.065 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:30000 IOs, 12277269504 bytes 2026-02-22T21:39:03.121 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:30100 IOs, 12359670784 bytes 2026-02-22T21:39:03.160 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:30200 IOs, 12401854464 bytes 2026-02-22T21:39:03.209 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:30300 IOs, 12473595904 bytes 2026-02-22T21:39:03.237 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:30400 IOs, 12486525952 bytes 2026-02-22T21:39:03.290 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:30500 IOs, 12563636224 bytes 2026-02-22T21:39:03.362 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:30600 IOs, 12678828032 bytes 2026-02-22T21:39:03.394 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:30700 IOs, 12699872256 bytes 2026-02-22T21:39:03.434 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:30800 IOs, 12750278656 bytes 2026-02-22T21:39:03.473 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:30900 IOs, 12791010304 bytes 2026-02-22T21:39:03.506 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:31000 IOs, 12821722112 bytes 2026-02-22T21:39:03.566 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:31100 IOs, 12911319040 bytes 2026-02-22T21:39:03.619 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:31200 IOs, 12988188672 bytes 2026-02-22T21:39:03.666 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:31300 IOs, 13061760000 bytes 2026-02-22T21:39:03.704 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:31400 IOs, 13089633280 bytes 2026-02-22T21:39:03.732 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:31500 IOs, 13106620416 bytes 2026-02-22T21:39:03.778 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:31600 IOs, 13160512512 bytes 2026-02-22T21:39:03.819 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:31700 IOs, 13214866432 bytes 2026-02-22T21:39:03.853 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:31800 IOs, 13243532288 bytes 2026-02-22T21:39:03.913 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:31900 IOs, 13338880000 bytes 2026-02-22T21:39:03.950 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:32000 IOs, 13361518592 bytes 2026-02-22T21:39:03.980 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:32100 IOs, 13392214016 bytes 2026-02-22T21:39:04.023 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:32200 IOs, 13463040000 bytes 2026-02-22T21:39:04.076 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:32300 IOs, 13519228928 bytes 2026-02-22T21:39:04.146 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:32400 IOs, 13639232512 bytes 2026-02-22T21:39:04.185 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:32500 IOs, 13696658432 bytes 2026-02-22T21:39:04.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:32600 IOs, 13702832128 bytes 2026-02-22T21:39:04.230 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:32700 IOs, 13704953856 bytes 2026-02-22T21:39:04.290 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:32800 IOs, 13803929600 bytes 2026-02-22T21:39:04.347 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:32900 IOs, 13880705024 bytes 2026-02-22T21:39:04.399 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:33000 IOs, 13953224704 bytes 2026-02-22T21:39:04.430 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:33100 IOs, 14000700416 bytes 2026-02-22T21:39:04.482 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:33200 IOs, 14059151360 bytes 2026-02-22T21:39:04.518 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:33300 IOs, 14091336704 bytes 2026-02-22T21:39:04.742 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:33400 IOs, 14117284864 bytes 2026-02-22T21:39:04.777 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:33500 IOs, 14153412608 bytes 2026-02-22T21:39:04.823 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:33600 IOs, 14211850240 bytes 2026-02-22T21:39:04.855 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:33700 IOs, 14235785216 bytes 2026-02-22T21:39:04.924 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:33800 IOs, 14349144064 bytes 2026-02-22T21:39:04.951 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:33900 IOs, 14374393856 bytes 2026-02-22T21:39:04.978 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:34000 IOs, 14386002944 bytes 2026-02-22T21:39:05.017 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:34100 IOs, 14434115584 bytes 2026-02-22T21:39:05.067 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:34200 IOs, 14491056128 bytes 2026-02-22T21:39:05.095 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:34300 IOs, 14522052608 bytes 2026-02-22T21:39:05.135 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:34400 IOs, 14556507136 bytes 2026-02-22T21:39:05.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:34500 IOs, 14578924544 bytes 2026-02-22T21:39:05.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:34600 IOs, 14611911680 bytes 2026-02-22T21:39:05.229 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:34700 IOs, 14637938688 bytes 2026-02-22T21:39:05.266 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:34800 IOs, 14675699712 bytes 2026-02-22T21:39:05.301 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:34900 IOs, 14713609216 bytes 2026-02-22T21:39:05.348 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:35000 IOs, 14763407360 bytes 2026-02-22T21:39:05.384 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:35100 IOs, 14811379712 bytes 2026-02-22T21:39:05.414 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:35200 IOs, 14816885760 bytes 2026-02-22T21:39:05.462 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:35300 IOs, 14889605120 bytes 2026-02-22T21:39:05.491 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:35400 IOs, 14917812224 bytes 2026-02-22T21:39:05.525 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:35500 IOs, 14945502208 bytes 2026-02-22T21:39:05.582 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:35600 IOs, 15042276352 bytes 2026-02-22T21:39:05.639 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:35700 IOs, 15114008576 bytes 2026-02-22T21:39:05.673 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:35800 IOs, 15146725376 bytes 2026-02-22T21:39:05.724 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:35900 IOs, 15221254144 bytes 2026-02-22T21:39:05.787 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:36000 IOs, 15331301376 bytes 2026-02-22T21:39:05.832 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:36100 IOs, 15383287808 bytes 2026-02-22T21:39:05.866 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:36200 IOs, 15413594112 bytes 2026-02-22T21:39:05.916 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:36300 IOs, 15486569472 bytes 2026-02-22T21:39:05.955 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:36400 IOs, 15519648768 bytes 2026-02-22T21:39:06.003 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:36500 IOs, 15590773760 bytes 2026-02-22T21:39:06.034 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:36600 IOs, 15615754240 bytes 2026-02-22T21:39:06.064 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:36700 IOs, 15659940864 bytes 2026-02-22T21:39:06.114 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:36800 IOs, 15709089792 bytes 2026-02-22T21:39:06.147 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:36900 IOs, 15746783232 bytes 2026-02-22T21:39:06.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:37000 IOs, 15788577792 bytes 2026-02-22T21:39:06.244 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:37100 IOs, 15862681600 bytes 2026-02-22T21:39:06.308 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:37200 IOs, 15967923200 bytes 2026-02-22T21:39:06.355 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:37300 IOs, 16030389248 bytes 2026-02-22T21:39:06.408 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:37400 IOs, 16107750400 bytes 2026-02-22T21:39:06.464 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ error_code=124 2026-02-22T21:39:06.465 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set -e 2026-02-22T21:39:06.465 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' 124 -eq 124 ']' 2026-02-22T21:39:06.465 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:39:06.465 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T21:39:06.465 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:39:06.465 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:39:06.465 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:39:06.465 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state_pattern=up+replaying 2026-02-22T21:39:06.465 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local description_pattern= 2026-02-22T21:39:06.465 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local service_pattern= 2026-02-22T21:39:06.465 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:39:06.465 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T21:39:07.466 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T21:39:07.466 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:39:07.466 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:39:07.466 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:39:07.466 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state_pattern=up+replaying 2026-02-22T21:39:07.466 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local description_pattern= 2026-02-22T21:39:07.466 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local service_pattern= 2026-02-22T21:39:07.466 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status 2026-02-22T21:39:07.466 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ CEPH_ARGS= 2026-02-22T21:39:07.466 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:39:07.500 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ status='test: 2026-02-22T21:39:07.500 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: global_id: b410963b-cae9-4337-a6ef-da65321c9374 2026-02-22T21:39:07.500 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: state: up+replaying 2026-02-22T21:39:07.500 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: description: replaying, {"bytes_per_second":17761792.0,"bytes_per_snapshot":266426880.0,"last_snapshot_bytes":532853760,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771796327,"remote_snapshot_timestamp":1771796327,"replay_state":"idle"} 2026-02-22T21:39:07.500 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: service: mirror.0 on trial026 2026-02-22T21:39:07.500 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: last_update: 2026-02-22 21:39:02 2026-02-22T21:39:07.500 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: peer_sites: 2026-02-22T21:39:07.500 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: name: cluster2 2026-02-22T21:39:07.500 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: state: down+unknown 2026-02-22T21:39:07.500 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: description: status not found 2026-02-22T21:39:07.500 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: last_update: ' 2026-02-22T21:39:07.500 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T21:39:07.501 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: state: up+replaying 2026-02-22T21:39:07.501 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep '^ description: .*' 2026-02-22T21:39:07.502 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: description: replaying, {"bytes_per_second":17761792.0,"bytes_per_snapshot":266426880.0,"last_snapshot_bytes":532853760,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771796327,"remote_snapshot_timestamp":1771796327,"replay_state":"idle"} 2026-02-22T21:39:07.502 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n '' ']' 2026-02-22T21:39:07.502 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ echo up+replaying 2026-02-22T21:39:07.502 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep '^up+' 2026-02-22T21:39:07.503 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:up+replaying 2026-02-22T21:39:07.503 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:39:07.504 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: service: mirror.0 on trial026 2026-02-22T21:39:07.504 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local last_update 2026-02-22T21:39:07.504 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:39:07.505 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ last_update='2026-02-22 21:39:02' 2026-02-22T21:39:07.506 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 21:39:02' 2026-02-22T21:39:07.506 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:39:07.506 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:39:07.506 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:39:07.506 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state_pattern=up+replaying 2026-02-22T21:39:07.506 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local 'prev_last_update=2026-02-22 21:39:02' 2026-02-22T21:39:07.506 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status 2026-02-22T21:39:07.506 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ CEPH_ARGS= 2026-02-22T21:39:07.506 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:39:07.537 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ status='WARNINGOKWARNING142074258mirror.0trial02619.2.3-689-gf90f4eaetrueOK42104261mirror.1trial02619.2.3-689-gf90f4eaefalseOK42144294mirror.2trial02619.2.3-689-gf90f4eaefalseOK42184265mirror.3trial02619.2.3-689-gf90f4eaefalseOKtestb410963b-cae9-4337-a6ef-da65321c9374up+replayingreplaying, {"bytes_per_second":17761792.0,"bytes_per_snapshot":266426880.0,"last_snapshot_bytes":532853760,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771796327,"remote_snapshot_timestamp":1771796327,"replay_state":"idle"}42074258mirror.0trial0262026-02-22 21:39:02cluster2cfa81703-817a-4074-adbc-5e6e200790bfdown+unknownstatus not found' 2026-02-22T21:39:07.537 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local last_update state 2026-02-22T21:39:07.537 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:39:07.539 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ last_update='2026-02-22 21:39:02' 2026-02-22T21:39:07.539 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:39:07.540 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ state=up+replaying 2026-02-22T21:39:07.540 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ echo up+replaying 2026-02-22T21:39:07.540 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep up+replaying 2026-02-22T21:39:07.541 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:up+replaying 2026-02-22T21:39:07.542 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:39:07.542 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:39:07.542 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ snap_name=snap2 2026-02-22T21:39:07.542 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_snap cluster2 mirror test snap2 2026-02-22T21:39:07.542 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:39:07.542 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:39:07.543 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:39:07.543 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap2 2026-02-22T21:39:07.543 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap2 --debug-rbd=20 --debug-journaler=20 2026-02-22T21:39:36.015 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T21:39:36.015 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:39:36.016 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:39:36.016 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:39:36.016 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T21:39:36.016 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:39:36.016 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:39:36.016 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:39:36.016 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=started 2026-02-22T21:39:36.016 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T21:39:36.016 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:39:36.016 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T21:39:37.016 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T21:39:37.016 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:39:37.016 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:39:37.016 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:39:37.016 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=started 2026-02-22T21:39:37.016 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_result 2026-02-22T21:39:37.016 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=stopped 2026-02-22T21:39:37.016 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:39:37.016 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ grep -i state 2026-02-22T21:39:37.017 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_instance=cluster1 2026-02-22T21:39:37.017 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:39:37.017 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T21:39:37.017 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance=cluster1 2026-02-22T21:39:37.017 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local loop_instance 2026-02-22T21:39:37.017 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:39:37.017 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sleep 0 2026-02-22T21:39:37.017 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:39:37.017 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ seq 0 3 2026-02-22T21:39:37.019 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:39:37.019 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:39:37.019 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:0 2026-02-22T21:39:37.019 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:39:37.019 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:39:37.019 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:39:37.019 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:0 2026-02-22T21:39:37.019 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:39:37.019 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:39:37.019 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:39:37.019 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=0 2026-02-22T21:39:37.019 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:39:37.019 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:39:37.019 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:39:37.019 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=0 2026-02-22T21:39:37.019 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T21:39:37.019 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:39:37.019 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:39:37.019 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:39:37.019 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:39:37.019 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:39:37.019 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:0 2026-02-22T21:39:37.019 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:39:37.019 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:39:37.020 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:39:37.020 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T21:39:37.020 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:39:37.020 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:39:37.020 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:39:37.020 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=0 2026-02-22T21:39:37.020 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=0 2026-02-22T21:39:37.020 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:39:37.031 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:39:37.032 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:39:37.032 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:39:37.032 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:39:37.077 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ return 0 2026-02-22T21:39:37.077 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:39:37.078 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:39:37.078 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep -i Replaying 2026-02-22T21:39:37.079 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: "state": "Replaying" 2026-02-22T21:39:37.079 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_state=started 2026-02-22T21:39:37.079 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test started = started 2026-02-22T21:39:37.079 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:39:37.079 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2026-02-22T21:39:37.079 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T21:39:37.079 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:39:37.079 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:39:37.079 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:39:37.079 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = journal ']' 2026-02-22T21:39:37.080 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T21:39:37.080 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-22T21:39:37.080 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:39:37.080 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:39:37.080 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:39:37.080 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-22T21:39:39.018 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Snapshot ID: 8 2026-02-22T21:39:39.024 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test 2026-02-22T21:39:39.024 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T21:39:39.024 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:39:39.024 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:39:39.024 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:39:39.024 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster2-mirror-test.status 2026-02-22T21:39:39.024 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster2-mirror-test.status 2026-02-22T21:39:39.024 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T21:39:39.025 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_log=/tmp/tmp.j44LZPowal/cluster2-mirror-test.status 2026-02-22T21:39:39.026 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster1-mirror-test.status 2026-02-22T21:39:39.026 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster1-mirror-test.status 2026-02-22T21:39:39.026 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T21:39:39.027 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_status_log=/tmp/tmp.j44LZPowal/cluster1-mirror-test.status 2026-02-22T21:39:39.027 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.j44LZPowal/cluster2-mirror-test.status 2026-02-22T21:39:39.027 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:39:39.027 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:39:39.028 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:39:39.028 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster2-mirror-test.status 2026-02-22T21:39:39.028 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-22T21:39:39.028 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:39:39.055 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T21:39:39.057 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snapshot_id=8 2026-02-22T21:39:39.057 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:39:39.057 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:39:39.057 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.2 2026-02-22T21:39:39.258 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.j44LZPowal/cluster1-mirror-test.status 2026-02-22T21:39:39.258 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:39:39.258 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:39:39.258 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:39:39.258 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-test.status 2026-02-22T21:39:39.258 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T21:39:39.258 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:39:39.287 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:39:39.287 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:39:39.288 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T21:39:39.288 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T21:39:39.288 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T21:39:39.288 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T21:39:39.288 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 8 = '' 2026-02-22T21:39:39.289 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:39:39.289 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.4 2026-02-22T21:39:39.689 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.j44LZPowal/cluster1-mirror-test.status 2026-02-22T21:39:39.690 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:39:39.690 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:39:39.690 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:39:39.690 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-test.status 2026-02-22T21:39:39.690 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T21:39:39.690 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:39:39.721 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:39:39.721 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:39:39.722 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T21:39:39.722 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T21:39:39.722 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T21:39:39.723 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T21:39:39.723 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 8 = '' 2026-02-22T21:39:39.723 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:39:39.723 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.8 2026-02-22T21:39:40.524 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.j44LZPowal/cluster1-mirror-test.status 2026-02-22T21:39:40.524 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:39:40.524 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:39:40.525 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:39:40.525 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-test.status 2026-02-22T21:39:40.525 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T21:39:40.525 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:39:40.577 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:39:40.577 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:39:40.578 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T21:39:40.579 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T21:39:40.579 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T21:39:40.579 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T21:39:40.579 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 8 = '' 2026-02-22T21:39:40.580 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:39:40.580 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1.6 2026-02-22T21:39:42.180 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.j44LZPowal/cluster1-mirror-test.status 2026-02-22T21:39:42.180 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:39:42.180 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:39:42.180 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:39:42.180 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-test.status 2026-02-22T21:39:42.180 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T21:39:42.180 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:39:42.213 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:39:42.215 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id=8 2026-02-22T21:39:42.215 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 8 = 8 2026-02-22T21:39:42.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:39:42.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snap_present cluster1 mirror test snap2 2026-02-22T21:39:42.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:39:42.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:39:42.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:39:42.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap2 2026-02-22T21:39:42.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T21:39:42.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T21:39:42.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T21:39:43.215 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 info mirror/test@snap2 2026-02-22T21:39:43.242 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'test': 2026-02-22T21:39:43.242 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 512 MiB in 128 objects 2026-02-22T21:39:43.242 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T21:39:43.242 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 4 2026-02-22T21:39:43.242 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 10a25aa14179 2026-02-22T21:39:43.242 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.10a25aa14179 2026-02-22T21:39:43.243 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T21:39:43.243 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock, non-primary 2026-02-22T21:39:43.243 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T21:39:43.243 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T21:39:43.243 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:38:05 2026 2026-02-22T21:39:43.243 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:38:05 2026 2026-02-22T21:39:43.243 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:38:05 2026 2026-02-22T21:39:43.243 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: protected: False 2026-02-22T21:39:43.243 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T21:39:43.243 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T21:39:43.243 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: b410963b-cae9-4337-a6ef-da65321c9374 2026-02-22T21:39:43.243 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: false 2026-02-22T21:39:43.244 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:39:43.244 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n snap1-clean ']' 2026-02-22T21:39:43.244 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ compare_image_snaps mirror test snap1-clean 2026-02-22T21:39:43.244 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:39:43.244 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:39:43.244 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap1-clean 2026-02-22T21:39:43.244 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local ret=0 2026-02-22T21:39:43.245 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local rmt_export=/tmp/tmp.j44LZPowal/cluster2-mirror-test.export 2026-02-22T21:39:43.245 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local loc_export=/tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:39:43.245 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:39:43.245 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror export test@snap1-clean /tmp/tmp.j44LZPowal/cluster2-mirror-test.export 2026-02-22T21:39:43.570 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-22T21:39:43.581 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 -p mirror export test@snap1-clean /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:39:43.915 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-22T21:39:43.927 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ cmp /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:39:44.106 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:39:44.229 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:39:44.230 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ compare_image_snaps mirror test snap2 2026-02-22T21:39:44.230 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:39:44.230 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:39:44.230 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap2 2026-02-22T21:39:44.230 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local ret=0 2026-02-22T21:39:44.230 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local rmt_export=/tmp/tmp.j44LZPowal/cluster2-mirror-test.export 2026-02-22T21:39:44.230 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local loc_export=/tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:39:44.230 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:39:44.230 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror export test@snap2 /tmp/tmp.j44LZPowal/cluster2-mirror-test.export 2026-02-22T21:39:44.561 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-22T21:39:44.569 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 -p mirror export test@snap2 /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:39:44.955 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-22T21:39:44.965 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ cmp /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:39:45.148 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:39:45.275 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:39:45.275 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ clean_snap_name=snap2-clean 2026-02-22T21:39:45.275 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_snap cluster2 mirror test snap2-clean 2026-02-22T21:39:45.275 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:39:45.275 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:39:45.275 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:39:45.275 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap2-clean 2026-02-22T21:39:45.275 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap2-clean --debug-rbd=20 --debug-journaler=20 2026-02-22T21:39:46.085 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 10` 2026-02-22T21:39:46.086 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ stress_write_image cluster2 mirror test 2026-02-22T21:39:46.086 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:39:46.086 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:39:46.086 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:39:46.086 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-22T21:39:46.087 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local duration=13 2026-02-22T21:39:46.087 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set +e 2026-02-22T21:39:46.087 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ timeout 13s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2026-02-22T21:39:46.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:100 IOs, 31614976 bytes 2026-02-22T21:39:46.220 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:200 IOs, 66508800 bytes 2026-02-22T21:39:46.263 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:300 IOs, 95752192 bytes 2026-02-22T21:39:46.306 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:400 IOs, 141239296 bytes 2026-02-22T21:39:46.350 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:500 IOs, 206255104 bytes 2026-02-22T21:39:46.388 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:600 IOs, 249809920 bytes 2026-02-22T21:39:46.426 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:700 IOs, 289031168 bytes 2026-02-22T21:39:46.459 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:800 IOs, 322138112 bytes 2026-02-22T21:39:46.494 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:900 IOs, 352365568 bytes 2026-02-22T21:39:46.542 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1000 IOs, 433633280 bytes 2026-02-22T21:39:46.590 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1100 IOs, 513667072 bytes 2026-02-22T21:39:46.634 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1200 IOs, 560491520 bytes 2026-02-22T21:39:46.663 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1300 IOs, 583641088 bytes 2026-02-22T21:39:46.699 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1400 IOs, 627429376 bytes 2026-02-22T21:39:46.730 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1500 IOs, 656860160 bytes 2026-02-22T21:39:46.750 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1600 IOs, 658764800 bytes 2026-02-22T21:39:46.772 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1700 IOs, 684843008 bytes 2026-02-22T21:39:46.831 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1800 IOs, 765673472 bytes 2026-02-22T21:39:46.898 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1900 IOs, 872210432 bytes 2026-02-22T21:39:46.946 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2000 IOs, 937535488 bytes 2026-02-22T21:39:46.981 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2100 IOs, 968228864 bytes 2026-02-22T21:39:47.003 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2200 IOs, 983081984 bytes 2026-02-22T21:39:47.059 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2300 IOs, 1072051200 bytes 2026-02-22T21:39:47.101 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2400 IOs, 1128066048 bytes 2026-02-22T21:39:47.126 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2500 IOs, 1140179968 bytes 2026-02-22T21:39:47.171 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2600 IOs, 1211441152 bytes 2026-02-22T21:39:47.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2700 IOs, 1220872192 bytes 2026-02-22T21:39:47.220 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2800 IOs, 1246583808 bytes 2026-02-22T21:39:47.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2900 IOs, 1317932032 bytes 2026-02-22T21:39:47.310 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3000 IOs, 1354544128 bytes 2026-02-22T21:39:47.351 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3100 IOs, 1416718336 bytes 2026-02-22T21:39:47.408 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3200 IOs, 1490413568 bytes 2026-02-22T21:39:47.461 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3300 IOs, 1574509568 bytes 2026-02-22T21:39:47.495 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3400 IOs, 1609001984 bytes 2026-02-22T21:39:47.527 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3500 IOs, 1638645760 bytes 2026-02-22T21:39:47.544 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3600 IOs, 1640209408 bytes 2026-02-22T21:39:47.569 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3700 IOs, 1659136000 bytes 2026-02-22T21:39:47.611 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3800 IOs, 1706510336 bytes 2026-02-22T21:39:47.655 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3900 IOs, 1776248832 bytes 2026-02-22T21:39:47.692 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4000 IOs, 1806528512 bytes 2026-02-22T21:39:47.732 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4100 IOs, 1855420416 bytes 2026-02-22T21:39:47.777 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4200 IOs, 1933366272 bytes 2026-02-22T21:39:47.811 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4300 IOs, 1946417152 bytes 2026-02-22T21:39:47.836 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4400 IOs, 1968692224 bytes 2026-02-22T21:39:47.857 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4500 IOs, 1969312768 bytes 2026-02-22T21:39:47.900 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4600 IOs, 2027070464 bytes 2026-02-22T21:39:47.939 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4700 IOs, 2089229312 bytes 2026-02-22T21:39:48.001 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4800 IOs, 2177260544 bytes 2026-02-22T21:39:48.040 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4900 IOs, 2212098048 bytes 2026-02-22T21:39:48.083 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5000 IOs, 2265286656 bytes 2026-02-22T21:39:48.111 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5100 IOs, 2311049216 bytes 2026-02-22T21:39:48.159 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5200 IOs, 2345768960 bytes 2026-02-22T21:39:48.195 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5300 IOs, 2385756160 bytes 2026-02-22T21:39:48.234 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5400 IOs, 2445786112 bytes 2026-02-22T21:39:48.260 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5500 IOs, 2454106112 bytes 2026-02-22T21:39:48.304 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5600 IOs, 2505357312 bytes 2026-02-22T21:39:48.339 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5700 IOs, 2552139776 bytes 2026-02-22T21:39:48.361 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5800 IOs, 2554268672 bytes 2026-02-22T21:39:48.399 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5900 IOs, 2602347520 bytes 2026-02-22T21:39:48.428 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6000 IOs, 2625700864 bytes 2026-02-22T21:39:48.479 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6100 IOs, 2714177536 bytes 2026-02-22T21:39:48.524 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6200 IOs, 2770983936 bytes 2026-02-22T21:39:48.544 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6300 IOs, 2775685120 bytes 2026-02-22T21:39:48.592 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6400 IOs, 2840802304 bytes 2026-02-22T21:39:48.625 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6500 IOs, 2878347264 bytes 2026-02-22T21:39:48.675 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6600 IOs, 2958052352 bytes 2026-02-22T21:39:48.711 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6700 IOs, 3006028800 bytes 2026-02-22T21:39:48.749 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6800 IOs, 3033452544 bytes 2026-02-22T21:39:48.781 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6900 IOs, 3075263488 bytes 2026-02-22T21:39:48.845 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7000 IOs, 3168429056 bytes 2026-02-22T21:39:48.877 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7100 IOs, 3194070016 bytes 2026-02-22T21:39:48.898 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7200 IOs, 3194694656 bytes 2026-02-22T21:39:48.942 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7300 IOs, 3257901056 bytes 2026-02-22T21:39:48.982 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7400 IOs, 3315150848 bytes 2026-02-22T21:39:49.223 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7500 IOs, 3379506176 bytes 2026-02-22T21:39:49.259 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7600 IOs, 3422270464 bytes 2026-02-22T21:39:49.288 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7700 IOs, 3455547392 bytes 2026-02-22T21:39:49.323 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7800 IOs, 3482972160 bytes 2026-02-22T21:39:49.362 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7900 IOs, 3515710464 bytes 2026-02-22T21:39:49.396 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8000 IOs, 3542944768 bytes 2026-02-22T21:39:49.446 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8100 IOs, 3615779840 bytes 2026-02-22T21:39:49.494 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8200 IOs, 3692054528 bytes 2026-02-22T21:39:49.517 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8300 IOs, 3695612928 bytes 2026-02-22T21:39:49.545 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8400 IOs, 3713084416 bytes 2026-02-22T21:39:49.585 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8500 IOs, 3761446912 bytes 2026-02-22T21:39:49.610 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8600 IOs, 3774565376 bytes 2026-02-22T21:39:49.650 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8700 IOs, 3807864832 bytes 2026-02-22T21:39:49.688 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8800 IOs, 3846461440 bytes 2026-02-22T21:39:49.710 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8900 IOs, 3858127872 bytes 2026-02-22T21:39:49.745 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9000 IOs, 3885623296 bytes 2026-02-22T21:39:49.779 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9100 IOs, 3923444736 bytes 2026-02-22T21:39:49.848 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9200 IOs, 4036466688 bytes 2026-02-22T21:39:49.892 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9300 IOs, 4080771072 bytes 2026-02-22T21:39:49.945 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9400 IOs, 4167848960 bytes 2026-02-22T21:39:49.983 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9500 IOs, 4203814912 bytes 2026-02-22T21:39:50.025 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9600 IOs, 4264424448 bytes 2026-02-22T21:39:50.057 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9700 IOs, 4302167040 bytes 2026-02-22T21:39:50.107 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9800 IOs, 4379560960 bytes 2026-02-22T21:39:50.144 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9900 IOs, 4417084416 bytes 2026-02-22T21:39:50.187 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10000 IOs, 4471796736 bytes 2026-02-22T21:39:50.219 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10100 IOs, 4509330432 bytes 2026-02-22T21:39:50.268 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10200 IOs, 4568534016 bytes 2026-02-22T21:39:50.289 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10300 IOs, 4585749504 bytes 2026-02-22T21:39:50.318 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10400 IOs, 4607040512 bytes 2026-02-22T21:39:50.357 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10500 IOs, 4636070912 bytes 2026-02-22T21:39:50.378 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10600 IOs, 4636663808 bytes 2026-02-22T21:39:50.422 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10700 IOs, 4693083136 bytes 2026-02-22T21:39:50.473 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10800 IOs, 4764218368 bytes 2026-02-22T21:39:50.512 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10900 IOs, 4808806400 bytes 2026-02-22T21:39:50.538 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11000 IOs, 4817389568 bytes 2026-02-22T21:39:50.594 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11100 IOs, 4887359488 bytes 2026-02-22T21:39:50.628 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11200 IOs, 4916045824 bytes 2026-02-22T21:39:50.668 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11300 IOs, 4967288832 bytes 2026-02-22T21:39:50.700 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11400 IOs, 4994203648 bytes 2026-02-22T21:39:50.743 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11500 IOs, 5045502976 bytes 2026-02-22T21:39:50.782 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11600 IOs, 5086296064 bytes 2026-02-22T21:39:50.818 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11700 IOs, 5124082688 bytes 2026-02-22T21:39:50.845 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11800 IOs, 5142306816 bytes 2026-02-22T21:39:50.890 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11900 IOs, 5193071616 bytes 2026-02-22T21:39:50.936 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12000 IOs, 5262521344 bytes 2026-02-22T21:39:50.963 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12100 IOs, 5269292032 bytes 2026-02-22T21:39:50.993 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12200 IOs, 5291707392 bytes 2026-02-22T21:39:51.041 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12300 IOs, 5370678272 bytes 2026-02-22T21:39:51.082 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12400 IOs, 5402199040 bytes 2026-02-22T21:39:51.130 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12500 IOs, 5462378496 bytes 2026-02-22T21:39:51.162 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12600 IOs, 5491148800 bytes 2026-02-22T21:39:51.183 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12700 IOs, 5499489280 bytes 2026-02-22T21:39:51.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12800 IOs, 5500108800 bytes 2026-02-22T21:39:51.246 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12900 IOs, 5558231040 bytes 2026-02-22T21:39:51.313 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13000 IOs, 5693597696 bytes 2026-02-22T21:39:51.366 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13100 IOs, 5741609984 bytes 2026-02-22T21:39:51.413 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13200 IOs, 5808494592 bytes 2026-02-22T21:39:51.451 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13300 IOs, 5853454336 bytes 2026-02-22T21:39:51.484 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13400 IOs, 5880247296 bytes 2026-02-22T21:39:51.538 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13500 IOs, 5957958656 bytes 2026-02-22T21:39:51.576 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13600 IOs, 6001738752 bytes 2026-02-22T21:39:51.608 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13700 IOs, 6028838912 bytes 2026-02-22T21:39:51.637 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13800 IOs, 6051511296 bytes 2026-02-22T21:39:51.664 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13900 IOs, 6066503680 bytes 2026-02-22T21:39:51.722 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14000 IOs, 6145062912 bytes 2026-02-22T21:39:51.749 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14100 IOs, 6160847872 bytes 2026-02-22T21:39:51.784 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14200 IOs, 6202714112 bytes 2026-02-22T21:39:51.823 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14300 IOs, 6243958784 bytes 2026-02-22T21:39:51.848 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14400 IOs, 6251501568 bytes 2026-02-22T21:39:51.907 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14500 IOs, 6350565376 bytes 2026-02-22T21:39:51.937 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14600 IOs, 6366962688 bytes 2026-02-22T21:39:51.984 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14700 IOs, 6420721664 bytes 2026-02-22T21:39:52.016 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14800 IOs, 6450509824 bytes 2026-02-22T21:39:52.045 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14900 IOs, 6479884288 bytes 2026-02-22T21:39:52.109 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15000 IOs, 6576618496 bytes 2026-02-22T21:39:52.144 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15100 IOs, 6614680576 bytes 2026-02-22T21:39:52.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15200 IOs, 6678398976 bytes 2026-02-22T21:39:52.230 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15300 IOs, 6728933376 bytes 2026-02-22T21:39:52.287 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15400 IOs, 6801076224 bytes 2026-02-22T21:39:52.341 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15500 IOs, 6896590848 bytes 2026-02-22T21:39:52.388 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15600 IOs, 6938450944 bytes 2026-02-22T21:39:52.433 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15700 IOs, 7007630336 bytes 2026-02-22T21:39:52.477 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15800 IOs, 7059123200 bytes 2026-02-22T21:39:52.512 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15900 IOs, 7090799616 bytes 2026-02-22T21:39:52.533 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16000 IOs, 7091412992 bytes 2026-02-22T21:39:52.566 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16100 IOs, 7128740864 bytes 2026-02-22T21:39:52.593 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16200 IOs, 7146866688 bytes 2026-02-22T21:39:52.648 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16300 IOs, 7224633344 bytes 2026-02-22T21:39:52.684 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16400 IOs, 7266994176 bytes 2026-02-22T21:39:52.725 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16500 IOs, 7307167744 bytes 2026-02-22T21:39:52.760 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16600 IOs, 7342626816 bytes 2026-02-22T21:39:52.788 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16700 IOs, 7366865920 bytes 2026-02-22T21:39:52.827 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16800 IOs, 7403575296 bytes 2026-02-22T21:39:52.869 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16900 IOs, 7460380672 bytes 2026-02-22T21:39:52.924 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17000 IOs, 7540911104 bytes 2026-02-22T21:39:52.966 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17100 IOs, 7577712640 bytes 2026-02-22T21:39:53.021 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17200 IOs, 7677561856 bytes 2026-02-22T21:39:53.069 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17300 IOs, 7731937280 bytes 2026-02-22T21:39:53.111 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17400 IOs, 7791330304 bytes 2026-02-22T21:39:53.150 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17500 IOs, 7808717824 bytes 2026-02-22T21:39:53.180 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17600 IOs, 7840249856 bytes 2026-02-22T21:39:53.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17700 IOs, 7856389120 bytes 2026-02-22T21:39:53.257 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17800 IOs, 7912641536 bytes 2026-02-22T21:39:53.305 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17900 IOs, 7986683904 bytes 2026-02-22T21:39:53.337 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18000 IOs, 8019521536 bytes 2026-02-22T21:39:53.381 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18100 IOs, 8070706176 bytes 2026-02-22T21:39:53.420 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18200 IOs, 8122238976 bytes 2026-02-22T21:39:53.455 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18300 IOs, 8137874432 bytes 2026-02-22T21:39:53.497 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18400 IOs, 8175878144 bytes 2026-02-22T21:39:53.533 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18500 IOs, 8217315328 bytes 2026-02-22T21:39:53.591 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18600 IOs, 8294858752 bytes 2026-02-22T21:39:53.623 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18700 IOs, 8315004928 bytes 2026-02-22T21:39:53.672 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18800 IOs, 8379352064 bytes 2026-02-22T21:39:53.720 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18900 IOs, 8439982080 bytes 2026-02-22T21:39:53.752 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19000 IOs, 8455717888 bytes 2026-02-22T21:39:53.791 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19100 IOs, 8499578880 bytes 2026-02-22T21:39:53.838 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19200 IOs, 8561272832 bytes 2026-02-22T21:39:53.893 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19300 IOs, 8664719360 bytes 2026-02-22T21:39:53.928 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19400 IOs, 8677923840 bytes 2026-02-22T21:39:53.967 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19500 IOs, 8718317568 bytes 2026-02-22T21:39:54.001 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19600 IOs, 8749372416 bytes 2026-02-22T21:39:54.022 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19700 IOs, 8749983744 bytes 2026-02-22T21:39:54.054 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19800 IOs, 8774721536 bytes 2026-02-22T21:39:54.107 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19900 IOs, 8838059008 bytes 2026-02-22T21:39:54.156 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20000 IOs, 8898534400 bytes 2026-02-22T21:39:54.215 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20100 IOs, 9003301888 bytes 2026-02-22T21:39:54.241 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20200 IOs, 9010199552 bytes 2026-02-22T21:39:54.287 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20300 IOs, 9057835008 bytes 2026-02-22T21:39:54.324 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20400 IOs, 9100863488 bytes 2026-02-22T21:39:54.357 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20500 IOs, 9127291904 bytes 2026-02-22T21:39:54.392 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20600 IOs, 9162374144 bytes 2026-02-22T21:39:54.418 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20700 IOs, 9171755008 bytes 2026-02-22T21:39:54.457 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20800 IOs, 9216366592 bytes 2026-02-22T21:39:54.500 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20900 IOs, 9266273280 bytes 2026-02-22T21:39:54.530 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21000 IOs, 9288015872 bytes 2026-02-22T21:39:54.608 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21100 IOs, 9410798592 bytes 2026-02-22T21:39:54.657 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21200 IOs, 9484328960 bytes 2026-02-22T21:39:54.705 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21300 IOs, 9546809344 bytes 2026-02-22T21:39:54.738 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21400 IOs, 9565857792 bytes 2026-02-22T21:39:54.761 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21500 IOs, 9573061632 bytes 2026-02-22T21:39:54.812 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21600 IOs, 9633660928 bytes 2026-02-22T21:39:54.859 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21700 IOs, 9691820032 bytes 2026-02-22T21:39:54.893 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21800 IOs, 9717252096 bytes 2026-02-22T21:39:54.936 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21900 IOs, 9758526464 bytes 2026-02-22T21:39:54.972 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22000 IOs, 9795483648 bytes 2026-02-22T21:39:55.022 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22100 IOs, 9871717376 bytes 2026-02-22T21:39:55.062 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22200 IOs, 9935555584 bytes 2026-02-22T21:39:55.102 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22300 IOs, 9964441600 bytes 2026-02-22T21:39:55.136 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22400 IOs, 9998473216 bytes 2026-02-22T21:39:55.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22500 IOs, 10056323072 bytes 2026-02-22T21:39:55.231 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22600 IOs, 10126976000 bytes 2026-02-22T21:39:55.260 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22700 IOs, 10148661248 bytes 2026-02-22T21:39:55.328 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22800 IOs, 10281154560 bytes 2026-02-22T21:39:55.367 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22900 IOs, 10303560704 bytes 2026-02-22T21:39:55.405 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23000 IOs, 10332713984 bytes 2026-02-22T21:39:55.440 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23100 IOs, 10361508864 bytes 2026-02-22T21:39:55.497 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23200 IOs, 10431373312 bytes 2026-02-22T21:39:55.536 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23300 IOs, 10476596224 bytes 2026-02-22T21:39:55.563 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23400 IOs, 10483156992 bytes 2026-02-22T21:39:55.584 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23500 IOs, 10483774464 bytes 2026-02-22T21:39:55.631 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23600 IOs, 10535964672 bytes 2026-02-22T21:39:55.668 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23700 IOs, 10575268864 bytes 2026-02-22T21:39:55.714 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23800 IOs, 10621045760 bytes 2026-02-22T21:39:55.762 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23900 IOs, 10686666752 bytes 2026-02-22T21:39:55.789 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24000 IOs, 10696560640 bytes 2026-02-22T21:39:55.835 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24100 IOs, 10754197504 bytes 2026-02-22T21:39:55.888 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24200 IOs, 10828146688 bytes 2026-02-22T21:39:55.918 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24300 IOs, 10854480896 bytes 2026-02-22T21:39:55.955 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24400 IOs, 10902264832 bytes 2026-02-22T21:39:56.016 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24500 IOs, 10984085504 bytes 2026-02-22T21:39:56.055 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24600 IOs, 10999000064 bytes 2026-02-22T21:39:56.083 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24700 IOs, 11020341248 bytes 2026-02-22T21:39:56.127 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24800 IOs, 11078888448 bytes 2026-02-22T21:39:56.155 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24900 IOs, 11098631168 bytes 2026-02-22T21:39:56.196 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25000 IOs, 11144294400 bytes 2026-02-22T21:39:56.235 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25100 IOs, 11181679616 bytes 2026-02-22T21:39:56.256 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25200 IOs, 11182295040 bytes 2026-02-22T21:39:56.290 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25300 IOs, 11209880576 bytes 2026-02-22T21:39:56.328 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25400 IOs, 11246485504 bytes 2026-02-22T21:39:56.365 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25500 IOs, 11288197120 bytes 2026-02-22T21:39:56.430 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25600 IOs, 11381699584 bytes 2026-02-22T21:39:56.454 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25700 IOs, 11387202560 bytes 2026-02-22T21:39:56.497 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25800 IOs, 11438188544 bytes 2026-02-22T21:39:56.543 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25900 IOs, 11492626432 bytes 2026-02-22T21:39:56.596 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:26000 IOs, 11568177152 bytes 2026-02-22T21:39:56.643 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:26100 IOs, 11631623168 bytes 2026-02-22T21:39:56.684 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:26200 IOs, 11666053120 bytes 2026-02-22T21:39:56.706 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:26300 IOs, 11666674688 bytes 2026-02-22T21:39:56.745 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:26400 IOs, 11710021632 bytes 2026-02-22T21:39:56.798 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:26500 IOs, 11782317056 bytes 2026-02-22T21:39:56.835 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:26600 IOs, 11833556992 bytes 2026-02-22T21:39:56.884 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:26700 IOs, 11886075904 bytes 2026-02-22T21:39:56.928 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:26800 IOs, 11938071552 bytes 2026-02-22T21:39:56.982 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:26900 IOs, 12007095296 bytes 2026-02-22T21:39:57.018 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:27000 IOs, 12041635840 bytes 2026-02-22T21:39:57.040 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:27100 IOs, 12042236928 bytes 2026-02-22T21:39:57.076 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:27200 IOs, 12095356928 bytes 2026-02-22T21:39:57.120 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:27300 IOs, 12131781632 bytes 2026-02-22T21:39:57.148 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:27400 IOs, 12167140352 bytes 2026-02-22T21:39:57.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:27500 IOs, 12212563968 bytes 2026-02-22T21:39:57.229 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:27600 IOs, 12228671488 bytes 2026-02-22T21:39:57.274 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:27700 IOs, 12276957184 bytes 2026-02-22T21:39:57.305 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:27800 IOs, 12298174464 bytes 2026-02-22T21:39:57.353 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:27900 IOs, 12353490944 bytes 2026-02-22T21:39:57.402 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:28000 IOs, 12407068672 bytes 2026-02-22T21:39:57.447 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:28100 IOs, 12456888320 bytes 2026-02-22T21:39:57.482 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:28200 IOs, 12485610496 bytes 2026-02-22T21:39:57.517 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:28300 IOs, 12512053248 bytes 2026-02-22T21:39:57.555 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:28400 IOs, 12557819904 bytes 2026-02-22T21:39:57.593 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:28500 IOs, 12592013312 bytes 2026-02-22T21:39:57.632 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:28600 IOs, 12628623360 bytes 2026-02-22T21:39:57.672 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:28700 IOs, 12671650816 bytes 2026-02-22T21:39:57.711 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:28800 IOs, 12695616512 bytes 2026-02-22T21:39:57.747 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:28900 IOs, 12726276096 bytes 2026-02-22T21:39:57.780 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:29000 IOs, 12771829760 bytes 2026-02-22T21:39:57.828 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:29100 IOs, 12813332480 bytes 2026-02-22T21:39:57.868 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:29200 IOs, 12858143744 bytes 2026-02-22T21:39:57.902 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:29300 IOs, 12890072064 bytes 2026-02-22T21:39:57.936 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:29400 IOs, 12915119104 bytes 2026-02-22T21:39:57.960 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:29500 IOs, 12924878848 bytes 2026-02-22T21:39:57.992 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:29600 IOs, 12949780480 bytes 2026-02-22T21:39:58.070 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:29700 IOs, 13090380800 bytes 2026-02-22T21:39:58.124 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:29800 IOs, 13166059520 bytes 2026-02-22T21:39:58.175 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:29900 IOs, 13224545280 bytes 2026-02-22T21:39:58.209 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:30000 IOs, 13268579328 bytes 2026-02-22T21:39:58.248 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:30100 IOs, 13291031552 bytes 2026-02-22T21:39:58.285 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:30200 IOs, 13329184768 bytes 2026-02-22T21:39:58.313 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:30300 IOs, 13341478912 bytes 2026-02-22T21:39:58.348 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:30400 IOs, 13391520768 bytes 2026-02-22T21:39:58.405 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:30500 IOs, 13463098368 bytes 2026-02-22T21:39:58.454 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:30600 IOs, 13514755072 bytes 2026-02-22T21:39:58.482 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:30700 IOs, 13531462656 bytes 2026-02-22T21:39:58.526 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:30800 IOs, 13579865088 bytes 2026-02-22T21:39:58.573 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:30900 IOs, 13649269760 bytes 2026-02-22T21:39:58.605 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:31000 IOs, 13671076864 bytes 2026-02-22T21:39:58.646 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:31100 IOs, 13708760064 bytes 2026-02-22T21:39:58.691 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:31200 IOs, 13756351488 bytes 2026-02-22T21:39:58.727 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:31300 IOs, 13783897088 bytes 2026-02-22T21:39:58.760 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:31400 IOs, 13810396160 bytes 2026-02-22T21:39:58.806 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:31500 IOs, 13867888640 bytes 2026-02-22T21:39:58.870 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:31600 IOs, 13944851456 bytes 2026-02-22T21:39:58.904 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:31700 IOs, 13978563584 bytes 2026-02-22T21:39:58.939 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:31800 IOs, 14014976000 bytes 2026-02-22T21:39:58.990 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:31900 IOs, 14063634432 bytes 2026-02-22T21:39:59.033 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:32000 IOs, 14116254720 bytes 2026-02-22T21:39:59.056 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:32100 IOs, 14130027520 bytes 2026-02-22T21:39:59.084 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:32200 IOs, 14130648064 bytes 2026-02-22T21:39:59.091 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ error_code=124 2026-02-22T21:39:59.091 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set -e 2026-02-22T21:39:59.091 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' 124 -eq 124 ']' 2026-02-22T21:39:59.091 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:39:59.091 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T21:39:59.091 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:39:59.091 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:39:59.091 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:39:59.091 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state_pattern=up+replaying 2026-02-22T21:39:59.091 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local description_pattern= 2026-02-22T21:39:59.091 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local service_pattern= 2026-02-22T21:39:59.091 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:39:59.091 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T21:40:00.092 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T21:40:00.092 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:40:00.092 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:40:00.092 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:40:00.092 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state_pattern=up+replaying 2026-02-22T21:40:00.092 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local description_pattern= 2026-02-22T21:40:00.092 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local service_pattern= 2026-02-22T21:40:00.093 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status 2026-02-22T21:40:00.093 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ CEPH_ARGS= 2026-02-22T21:40:00.093 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:40:00.126 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ status='test: 2026-02-22T21:40:00.126 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: global_id: b410963b-cae9-4337-a6ef-da65321c9374 2026-02-22T21:40:00.126 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: state: up+replaying 2026-02-22T21:40:00.126 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":266426880.0,"last_snapshot_bytes":532853760,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771796327,"remote_snapshot_timestamp":1771796327,"replay_state":"idle"} 2026-02-22T21:40:00.126 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: service: mirror.0 on trial026 2026-02-22T21:40:00.126 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: last_update: 2026-02-22 21:39:32 2026-02-22T21:40:00.126 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: peer_sites: 2026-02-22T21:40:00.126 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: name: cluster2 2026-02-22T21:40:00.126 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: state: up+stopped 2026-02-22T21:40:00.126 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: description: local image is primary 2026-02-22T21:40:00.126 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: last_update: 2026-02-22 21:39:33' 2026-02-22T21:40:00.126 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T21:40:00.126 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: state: up+replaying 2026-02-22T21:40:00.127 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep '^ description: .*' 2026-02-22T21:40:00.127 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":266426880.0,"last_snapshot_bytes":532853760,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771796327,"remote_snapshot_timestamp":1771796327,"replay_state":"idle"} 2026-02-22T21:40:00.128 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n '' ']' 2026-02-22T21:40:00.128 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ echo up+replaying 2026-02-22T21:40:00.128 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep '^up+' 2026-02-22T21:40:00.129 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:up+replaying 2026-02-22T21:40:00.129 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:40:00.130 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: service: mirror.0 on trial026 2026-02-22T21:40:00.130 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local last_update 2026-02-22T21:40:00.130 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:40:00.132 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ last_update='2026-02-22 21:39:32' 2026-02-22T21:40:00.132 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 21:39:32' 2026-02-22T21:40:00.132 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:40:00.132 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:40:00.132 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:40:00.132 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state_pattern=up+replaying 2026-02-22T21:40:00.132 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local 'prev_last_update=2026-02-22 21:39:32' 2026-02-22T21:40:00.132 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status 2026-02-22T21:40:00.132 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ CEPH_ARGS= 2026-02-22T21:40:00.132 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:40:00.163 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ status='OKOKOK142074258mirror.0trial02619.2.3-689-gf90f4eaetrueOK42104261mirror.1trial02619.2.3-689-gf90f4eaefalseOK42144294mirror.2trial02619.2.3-689-gf90f4eaefalseOK42184265mirror.3trial02619.2.3-689-gf90f4eaefalseOKtestb410963b-cae9-4337-a6ef-da65321c9374up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":266426880.0,"last_snapshot_bytes":532853760,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771796327,"remote_snapshot_timestamp":1771796327,"replay_state":"idle"}42074258mirror.0trial0262026-02-22 21:39:32cluster2cfa81703-817a-4074-adbc-5e6e200790bfup+stoppedlocal image is primary2026-02-22 21:39:33' 2026-02-22T21:40:00.163 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local last_update state 2026-02-22T21:40:00.163 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:40:00.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ last_update='2026-02-22 21:39:32' 2026-02-22T21:40:00.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:40:00.166 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ state=up+replaying 2026-02-22T21:40:00.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ echo up+replaying 2026-02-22T21:40:00.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep up+replaying 2026-02-22T21:40:00.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:up+replaying 2026-02-22T21:40:00.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:40:00.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:40:00.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ snap_name=snap3 2026-02-22T21:40:00.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_snap cluster2 mirror test snap3 2026-02-22T21:40:00.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:40:00.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:40:00.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:40:00.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap3 2026-02-22T21:40:00.169 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap3 --debug-rbd=20 --debug-journaler=20 2026-02-22T21:40:19.222 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.0 2026-02-22T21:40:19.222 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Sent signal 15 2026-02-22T21:40:19.222 INFO:tasks.rbd_mirror_thrash:waiting for 9.473462764184019 secs before reviving daemons 2026-02-22T21:40:19.223 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial026.stderr:2026-02-22T21:40:19.222+0000 7f4f1972b640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 (PID: 50795) UID: 1000 2026-02-22T21:40:21.679 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.9860430083164079) > weight (0.25) 2026-02-22T21:40:21.680 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.9769411159340575) > weight (0.25) 2026-02-22T21:40:21.680 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.2 2026-02-22T21:40:21.680 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Sent signal 15 2026-02-22T21:40:21.680 INFO:tasks.rbd_mirror_thrash:waiting for 6.840745373667897 secs before reviving daemons 2026-02-22T21:40:21.681 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial026.stderr:2026-02-22T21:40:21.679+0000 7fe0fde21640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.2 (PID: 48011) UID: 1000 2026-02-22T21:40:28.522 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.2 2026-02-22T21:40:28.522 INFO:teuthology.orchestra.run:waiting for 600 2026-02-22T21:40:28.522 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.2 2026-02-22T21:40:28.522 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Restarting daemon 2026-02-22T21:40:28.523 DEBUG:teuthology.orchestra.run.trial026:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.2 2026-02-22T21:40:28.525 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Started 2026-02-22T21:40:28.525 INFO:tasks.rbd_mirror_thrash:waiting for 64.46556250154056 secs before thrashing 2026-02-22T21:40:28.696 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.0 2026-02-22T21:40:28.697 INFO:teuthology.orchestra.run:waiting for 600 2026-02-22T21:40:28.697 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.0 2026-02-22T21:40:28.697 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Restarting daemon 2026-02-22T21:40:28.697 DEBUG:teuthology.orchestra.run.trial026:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 2026-02-22T21:40:28.700 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Started 2026-02-22T21:40:28.700 INFO:tasks.rbd_mirror_thrash:waiting for 117.28072063575613 secs before thrashing 2026-02-22T21:40:30.113 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T21:40:30.113 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:40:30.113 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:40:30.113 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:40:30.113 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T21:40:30.114 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:40:30.114 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:40:30.114 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:40:30.114 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=started 2026-02-22T21:40:30.114 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T21:40:30.114 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:40:30.114 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T21:40:31.114 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T21:40:31.115 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:40:31.115 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:40:31.115 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:40:31.115 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=started 2026-02-22T21:40:31.115 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_result 2026-02-22T21:40:31.115 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=stopped 2026-02-22T21:40:31.115 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:40:31.115 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ grep -i state 2026-02-22T21:40:31.116 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_instance=cluster1 2026-02-22T21:40:31.116 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:40:31.116 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T21:40:31.116 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance=cluster1 2026-02-22T21:40:31.116 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local loop_instance 2026-02-22T21:40:31.116 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:40:31.116 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sleep 0 2026-02-22T21:40:31.116 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:40:31.119 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ seq 0 3 2026-02-22T21:40:31.119 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:40:31.119 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:40:31.119 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:0 2026-02-22T21:40:31.119 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:40:31.120 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:40:31.120 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:40:31.120 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:0 2026-02-22T21:40:31.120 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:40:31.120 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:40:31.120 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:40:31.120 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=0 2026-02-22T21:40:31.120 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:40:31.120 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:40:31.120 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:40:31.120 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=0 2026-02-22T21:40:31.120 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T21:40:31.121 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:40:31.121 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:40:31.121 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:40:31.121 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:40:31.121 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:40:31.121 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:0 2026-02-22T21:40:31.121 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:40:31.121 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:40:31.121 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:40:31.121 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T21:40:31.121 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:40:31.122 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:40:31.122 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:40:31.122 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=0 2026-02-22T21:40:31.122 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=0 2026-02-22T21:40:31.122 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:40:31.129 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:40:31.129 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:40:31.130 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:40:31.130 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:40:31.175 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ return 0 2026-02-22T21:40:31.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:40:31.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:40:31.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep -i Replaying 2026-02-22T21:40:31.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: "state": "Replaying" 2026-02-22T21:40:31.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_state=started 2026-02-22T21:40:31.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test started = started 2026-02-22T21:40:31.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:40:31.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2026-02-22T21:40:31.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T21:40:31.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:40:31.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:40:31.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:40:31.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = journal ']' 2026-02-22T21:40:31.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T21:40:31.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-22T21:40:31.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:40:31.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:40:31.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:40:31.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-22T21:40:33.114 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Snapshot ID: 11 2026-02-22T21:40:33.124 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test 2026-02-22T21:40:33.124 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T21:40:33.124 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:40:33.125 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:40:33.125 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:40:33.125 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster2-mirror-test.status 2026-02-22T21:40:33.125 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster2-mirror-test.status 2026-02-22T21:40:33.125 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T21:40:33.126 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_log=/tmp/tmp.j44LZPowal/cluster2-mirror-test.status 2026-02-22T21:40:33.126 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster1-mirror-test.status 2026-02-22T21:40:33.126 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster1-mirror-test.status 2026-02-22T21:40:33.127 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T21:40:33.128 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_status_log=/tmp/tmp.j44LZPowal/cluster1-mirror-test.status 2026-02-22T21:40:33.128 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.j44LZPowal/cluster2-mirror-test.status 2026-02-22T21:40:33.129 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:40:33.129 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:40:33.129 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:40:33.129 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster2-mirror-test.status 2026-02-22T21:40:33.129 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-22T21:40:33.129 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:40:33.159 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T21:40:33.160 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snapshot_id=11 2026-02-22T21:40:33.160 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:40:33.160 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:40:33.160 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.2 2026-02-22T21:40:33.362 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.j44LZPowal/cluster1-mirror-test.status 2026-02-22T21:40:33.362 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:40:33.362 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:40:33.362 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:40:33.363 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-test.status 2026-02-22T21:40:33.363 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T21:40:33.363 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:40:33.392 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:40:33.392 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:40:33.393 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T21:40:33.393 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T21:40:33.393 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T21:40:33.393 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T21:40:33.393 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 11 = '' 2026-02-22T21:40:33.393 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:40:33.394 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.4 2026-02-22T21:40:33.795 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.j44LZPowal/cluster1-mirror-test.status 2026-02-22T21:40:33.795 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:40:33.795 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:40:33.795 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:40:33.795 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-test.status 2026-02-22T21:40:33.795 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T21:40:33.795 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:40:33.826 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:40:33.826 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:40:33.827 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T21:40:33.827 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T21:40:33.827 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T21:40:33.828 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T21:40:33.828 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 11 = '' 2026-02-22T21:40:33.828 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:40:33.829 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.8 2026-02-22T21:40:34.629 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.j44LZPowal/cluster1-mirror-test.status 2026-02-22T21:40:34.629 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:40:34.630 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:40:34.630 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:40:34.630 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-test.status 2026-02-22T21:40:34.630 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T21:40:34.630 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:40:34.661 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:40:34.663 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id=11 2026-02-22T21:40:34.663 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 11 = 11 2026-02-22T21:40:34.663 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:40:34.663 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snap_present cluster1 mirror test snap3 2026-02-22T21:40:34.663 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:40:34.663 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:40:34.663 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:40:34.663 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap3 2026-02-22T21:40:34.663 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T21:40:34.663 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T21:40:34.663 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T21:40:35.664 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 info mirror/test@snap3 2026-02-22T21:40:35.693 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'test': 2026-02-22T21:40:35.693 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 512 MiB in 128 objects 2026-02-22T21:40:35.693 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T21:40:35.693 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 6 2026-02-22T21:40:35.693 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 10a25aa14179 2026-02-22T21:40:35.693 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.10a25aa14179 2026-02-22T21:40:35.693 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T21:40:35.693 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock, non-primary 2026-02-22T21:40:35.693 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T21:40:35.693 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T21:40:35.693 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:38:05 2026 2026-02-22T21:40:35.693 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:39:43 2026 2026-02-22T21:40:35.693 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:38:05 2026 2026-02-22T21:40:35.693 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: protected: False 2026-02-22T21:40:35.693 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T21:40:35.693 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T21:40:35.693 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: b410963b-cae9-4337-a6ef-da65321c9374 2026-02-22T21:40:35.693 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: false 2026-02-22T21:40:35.694 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:40:35.695 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n snap2-clean ']' 2026-02-22T21:40:35.695 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ compare_image_snaps mirror test snap2-clean 2026-02-22T21:40:35.695 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:40:35.695 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:40:35.695 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap2-clean 2026-02-22T21:40:35.695 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local ret=0 2026-02-22T21:40:35.695 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local rmt_export=/tmp/tmp.j44LZPowal/cluster2-mirror-test.export 2026-02-22T21:40:35.695 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local loc_export=/tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:40:35.695 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:40:35.695 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror export test@snap2-clean /tmp/tmp.j44LZPowal/cluster2-mirror-test.export 2026-02-22T21:40:36.037 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-22T21:40:36.049 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 -p mirror export test@snap2-clean /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:40:36.461 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-22T21:40:36.472 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ cmp /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:40:36.651 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:40:36.773 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:40:36.774 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ compare_image_snaps mirror test snap3 2026-02-22T21:40:36.774 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:40:36.774 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:40:36.774 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap3 2026-02-22T21:40:36.774 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local ret=0 2026-02-22T21:40:36.774 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local rmt_export=/tmp/tmp.j44LZPowal/cluster2-mirror-test.export 2026-02-22T21:40:36.774 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local loc_export=/tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:40:36.774 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:40:36.774 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror export test@snap3 /tmp/tmp.j44LZPowal/cluster2-mirror-test.export 2026-02-22T21:40:37.105 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-22T21:40:37.114 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 -p mirror export test@snap3 /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:40:37.492 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-22T21:40:37.503 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ cmp /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:40:37.685 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:40:37.810 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:40:37.811 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ clean_snap_name=snap3-clean 2026-02-22T21:40:37.811 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_snap cluster2 mirror test snap3-clean 2026-02-22T21:40:37.811 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:40:37.811 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:40:37.811 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:40:37.811 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap3-clean 2026-02-22T21:40:37.811 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap3-clean --debug-rbd=20 --debug-journaler=20 2026-02-22T21:40:38.011 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 10` 2026-02-22T21:40:38.011 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ stress_write_image cluster2 mirror test 2026-02-22T21:40:38.011 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:40:38.012 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:40:38.012 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:40:38.012 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-22T21:40:38.013 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local duration=10 2026-02-22T21:40:38.013 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set +e 2026-02-22T21:40:38.014 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ timeout 10s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2026-02-22T21:40:38.119 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:100 IOs, 39597056 bytes 2026-02-22T21:40:38.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:200 IOs, 94367744 bytes 2026-02-22T21:40:38.223 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:300 IOs, 120913920 bytes 2026-02-22T21:40:38.280 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:400 IOs, 188549120 bytes 2026-02-22T21:40:38.331 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:500 IOs, 271492096 bytes 2026-02-22T21:40:38.399 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:600 IOs, 377720832 bytes 2026-02-22T21:40:38.435 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:700 IOs, 395742208 bytes 2026-02-22T21:40:38.466 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:800 IOs, 407165952 bytes 2026-02-22T21:40:38.498 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:900 IOs, 431930368 bytes 2026-02-22T21:40:38.538 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1000 IOs, 477339648 bytes 2026-02-22T21:40:38.583 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1100 IOs, 529221632 bytes 2026-02-22T21:40:38.631 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1200 IOs, 588241920 bytes 2026-02-22T21:40:38.694 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1300 IOs, 680175616 bytes 2026-02-22T21:40:38.729 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1400 IOs, 705485824 bytes 2026-02-22T21:40:38.769 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1500 IOs, 750141440 bytes 2026-02-22T21:40:38.832 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1600 IOs, 866454528 bytes 2026-02-22T21:40:38.892 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1700 IOs, 920236032 bytes 2026-02-22T21:40:38.919 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1800 IOs, 934960128 bytes 2026-02-22T21:40:38.960 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1900 IOs, 975364096 bytes 2026-02-22T21:40:39.012 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2000 IOs, 1043604480 bytes 2026-02-22T21:40:39.048 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2100 IOs, 1072278528 bytes 2026-02-22T21:40:39.090 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2200 IOs, 1113904128 bytes 2026-02-22T21:40:39.126 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2300 IOs, 1127931904 bytes 2026-02-22T21:40:39.159 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2400 IOs, 1148829696 bytes 2026-02-22T21:40:39.187 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2500 IOs, 1157579776 bytes 2026-02-22T21:40:39.240 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2600 IOs, 1218478080 bytes 2026-02-22T21:40:39.301 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2700 IOs, 1303940096 bytes 2026-02-22T21:40:39.337 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2800 IOs, 1323485184 bytes 2026-02-22T21:40:39.377 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2900 IOs, 1369397248 bytes 2026-02-22T21:40:39.425 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3000 IOs, 1429002240 bytes 2026-02-22T21:40:39.471 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3100 IOs, 1463526400 bytes 2026-02-22T21:40:39.509 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3200 IOs, 1498184704 bytes 2026-02-22T21:40:39.558 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3300 IOs, 1565449216 bytes 2026-02-22T21:40:39.587 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3400 IOs, 1569537024 bytes 2026-02-22T21:40:39.651 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3500 IOs, 1663009792 bytes 2026-02-22T21:40:39.725 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3600 IOs, 1781259264 bytes 2026-02-22T21:40:39.764 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3700 IOs, 1811223552 bytes 2026-02-22T21:40:39.786 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3800 IOs, 1811837952 bytes 2026-02-22T21:40:39.826 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3900 IOs, 1861936128 bytes 2026-02-22T21:40:39.876 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4000 IOs, 1899293696 bytes 2026-02-22T21:40:39.904 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4100 IOs, 1916518400 bytes 2026-02-22T21:40:39.952 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4200 IOs, 1963402240 bytes 2026-02-22T21:40:39.996 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4300 IOs, 2002780160 bytes 2026-02-22T21:40:40.038 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4400 IOs, 2034968576 bytes 2026-02-22T21:40:40.075 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4500 IOs, 2068577280 bytes 2026-02-22T21:40:40.135 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4600 IOs, 2168052736 bytes 2026-02-22T21:40:40.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4700 IOs, 2265226240 bytes 2026-02-22T21:40:40.239 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4800 IOs, 2288228352 bytes 2026-02-22T21:40:40.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4900 IOs, 2340078592 bytes 2026-02-22T21:40:40.322 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5000 IOs, 2363219968 bytes 2026-02-22T21:40:40.358 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5100 IOs, 2387472384 bytes 2026-02-22T21:40:40.407 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5200 IOs, 2443333632 bytes 2026-02-22T21:40:40.432 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5300 IOs, 2461677568 bytes 2026-02-22T21:40:40.471 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5400 IOs, 2487791616 bytes 2026-02-22T21:40:40.515 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5500 IOs, 2532711424 bytes 2026-02-22T21:40:40.555 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5600 IOs, 2567489536 bytes 2026-02-22T21:40:40.603 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5700 IOs, 2639297536 bytes 2026-02-22T21:40:40.637 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5800 IOs, 2649455616 bytes 2026-02-22T21:40:40.684 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5900 IOs, 2709245952 bytes 2026-02-22T21:40:40.730 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6000 IOs, 2734404608 bytes 2026-02-22T21:40:40.759 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6100 IOs, 2741965824 bytes 2026-02-22T21:40:40.790 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6200 IOs, 2755957760 bytes 2026-02-22T21:40:40.829 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6300 IOs, 2782912512 bytes 2026-02-22T21:40:40.880 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6400 IOs, 2843149312 bytes 2026-02-22T21:40:40.942 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6500 IOs, 2916678656 bytes 2026-02-22T21:40:40.974 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6600 IOs, 2929041408 bytes 2026-02-22T21:40:41.010 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6700 IOs, 2948324352 bytes 2026-02-22T21:40:41.059 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6800 IOs, 2995163136 bytes 2026-02-22T21:40:41.099 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6900 IOs, 3028610048 bytes 2026-02-22T21:40:41.136 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7000 IOs, 3069000704 bytes 2026-02-22T21:40:41.173 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7100 IOs, 3089883136 bytes 2026-02-22T21:40:41.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7200 IOs, 3094441984 bytes 2026-02-22T21:40:41.224 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7300 IOs, 3095036928 bytes 2026-02-22T21:40:41.261 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7400 IOs, 3127414784 bytes 2026-02-22T21:40:41.296 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7500 IOs, 3149307904 bytes 2026-02-22T21:40:41.334 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7600 IOs, 3197604864 bytes 2026-02-22T21:40:41.401 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7700 IOs, 3305833472 bytes 2026-02-22T21:40:41.454 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7800 IOs, 3352758272 bytes 2026-02-22T21:40:41.488 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7900 IOs, 3362473984 bytes 2026-02-22T21:40:41.535 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8000 IOs, 3415540736 bytes 2026-02-22T21:40:41.595 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8100 IOs, 3499754496 bytes 2026-02-22T21:40:41.641 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8200 IOs, 3550099456 bytes 2026-02-22T21:40:41.669 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8300 IOs, 3553876992 bytes 2026-02-22T21:40:41.710 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8400 IOs, 3590537216 bytes 2026-02-22T21:40:41.748 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8500 IOs, 3614298112 bytes 2026-02-22T21:40:41.783 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8600 IOs, 3647332352 bytes 2026-02-22T21:40:41.835 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8700 IOs, 3710511104 bytes 2026-02-22T21:40:41.863 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8800 IOs, 3720601600 bytes 2026-02-22T21:40:41.885 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8900 IOs, 3721195520 bytes 2026-02-22T21:40:41.919 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9000 IOs, 3746241536 bytes 2026-02-22T21:40:41.954 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9100 IOs, 3777798144 bytes 2026-02-22T21:40:41.998 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9200 IOs, 3823070208 bytes 2026-02-22T21:40:42.034 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9300 IOs, 3858162688 bytes 2026-02-22T21:40:42.089 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9400 IOs, 3931342848 bytes 2026-02-22T21:40:42.118 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9500 IOs, 3942248448 bytes 2026-02-22T21:40:42.183 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9600 IOs, 4031720448 bytes 2026-02-22T21:40:42.220 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9700 IOs, 4056100864 bytes 2026-02-22T21:40:42.250 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9800 IOs, 4097121280 bytes 2026-02-22T21:40:42.313 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9900 IOs, 4153656320 bytes 2026-02-22T21:40:42.352 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10000 IOs, 4191397888 bytes 2026-02-22T21:40:42.414 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10100 IOs, 4277071872 bytes 2026-02-22T21:40:42.446 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10200 IOs, 4297444352 bytes 2026-02-22T21:40:42.493 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10300 IOs, 4357703680 bytes 2026-02-22T21:40:42.560 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10400 IOs, 4461787136 bytes 2026-02-22T21:40:42.605 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10500 IOs, 4514866176 bytes 2026-02-22T21:40:42.649 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10600 IOs, 4551478272 bytes 2026-02-22T21:40:42.678 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10700 IOs, 4554741760 bytes 2026-02-22T21:40:42.720 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10800 IOs, 4594335744 bytes 2026-02-22T21:40:42.751 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10900 IOs, 4607925248 bytes 2026-02-22T21:40:42.796 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11000 IOs, 4673306624 bytes 2026-02-22T21:40:42.849 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11100 IOs, 4713205760 bytes 2026-02-22T21:40:42.885 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11200 IOs, 4735807488 bytes 2026-02-22T21:40:42.921 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11300 IOs, 4753014784 bytes 2026-02-22T21:40:42.954 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11400 IOs, 4773081088 bytes 2026-02-22T21:40:43.002 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11500 IOs, 4826749952 bytes 2026-02-22T21:40:43.058 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11600 IOs, 4888476672 bytes 2026-02-22T21:40:43.115 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11700 IOs, 4955788288 bytes 2026-02-22T21:40:43.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11800 IOs, 5021062144 bytes 2026-02-22T21:40:43.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11900 IOs, 5031717888 bytes 2026-02-22T21:40:43.230 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12000 IOs, 5058290688 bytes 2026-02-22T21:40:43.281 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12100 IOs, 5114346496 bytes 2026-02-22T21:40:43.319 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12200 IOs, 5139306496 bytes 2026-02-22T21:40:43.350 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12300 IOs, 5155731456 bytes 2026-02-22T21:40:43.388 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12400 IOs, 5185511424 bytes 2026-02-22T21:40:43.445 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12500 IOs, 5265182720 bytes 2026-02-22T21:40:43.490 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12600 IOs, 5303756800 bytes 2026-02-22T21:40:43.543 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12700 IOs, 5362649088 bytes 2026-02-22T21:40:43.577 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12800 IOs, 5389758464 bytes 2026-02-22T21:40:43.616 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12900 IOs, 5421933568 bytes 2026-02-22T21:40:43.674 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13000 IOs, 5487407104 bytes 2026-02-22T21:40:43.716 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13100 IOs, 5533773824 bytes 2026-02-22T21:40:43.754 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13200 IOs, 5558528000 bytes 2026-02-22T21:40:43.809 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13300 IOs, 5631612928 bytes 2026-02-22T21:40:43.848 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13400 IOs, 5666272256 bytes 2026-02-22T21:40:43.880 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13500 IOs, 5687515136 bytes 2026-02-22T21:40:43.917 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13600 IOs, 5715112960 bytes 2026-02-22T21:40:43.978 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13700 IOs, 5794531328 bytes 2026-02-22T21:40:44.012 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13800 IOs, 5813094400 bytes 2026-02-22T21:40:44.047 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13900 IOs, 5839066112 bytes 2026-02-22T21:40:44.074 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14000 IOs, 5848302592 bytes 2026-02-22T21:40:44.130 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14100 IOs, 5914536960 bytes 2026-02-22T21:40:44.184 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14200 IOs, 5972379648 bytes 2026-02-22T21:40:44.246 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14300 IOs, 6057761792 bytes 2026-02-22T21:40:44.327 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14400 IOs, 6190159872 bytes 2026-02-22T21:40:44.367 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14500 IOs, 6214665216 bytes 2026-02-22T21:40:44.414 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14600 IOs, 6255563776 bytes 2026-02-22T21:40:44.446 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14700 IOs, 6283174912 bytes 2026-02-22T21:40:44.496 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14800 IOs, 6345744384 bytes 2026-02-22T21:40:44.532 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14900 IOs, 6387957760 bytes 2026-02-22T21:40:44.561 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15000 IOs, 6390987776 bytes 2026-02-22T21:40:44.607 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15100 IOs, 6457324544 bytes 2026-02-22T21:40:44.657 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15200 IOs, 6510315520 bytes 2026-02-22T21:40:44.701 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15300 IOs, 6550344704 bytes 2026-02-22T21:40:44.738 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15400 IOs, 6573408256 bytes 2026-02-22T21:40:44.795 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15500 IOs, 6643270656 bytes 2026-02-22T21:40:44.853 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15600 IOs, 6719370240 bytes 2026-02-22T21:40:44.894 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15700 IOs, 6764845056 bytes 2026-02-22T21:40:44.936 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15800 IOs, 6804720640 bytes 2026-02-22T21:40:44.983 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15900 IOs, 6855816192 bytes 2026-02-22T21:40:45.038 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16000 IOs, 6923679744 bytes 2026-02-22T21:40:45.077 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16100 IOs, 6948959232 bytes 2026-02-22T21:40:45.120 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16200 IOs, 6979822592 bytes 2026-02-22T21:40:45.145 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16300 IOs, 6980418560 bytes 2026-02-22T21:40:45.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16400 IOs, 7073694720 bytes 2026-02-22T21:40:45.252 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16500 IOs, 7117103104 bytes 2026-02-22T21:40:45.304 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16600 IOs, 7177159680 bytes 2026-02-22T21:40:45.334 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16700 IOs, 7192318976 bytes 2026-02-22T21:40:45.369 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16800 IOs, 7215938560 bytes 2026-02-22T21:40:45.403 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16900 IOs, 7254837248 bytes 2026-02-22T21:40:45.462 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17000 IOs, 7319482368 bytes 2026-02-22T21:40:45.504 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17100 IOs, 7360006144 bytes 2026-02-22T21:40:45.549 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17200 IOs, 7402174464 bytes 2026-02-22T21:40:45.599 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17300 IOs, 7475337216 bytes 2026-02-22T21:40:45.636 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17400 IOs, 7519384576 bytes 2026-02-22T21:40:45.678 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17500 IOs, 7532878848 bytes 2026-02-22T21:40:45.721 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17600 IOs, 7604954112 bytes 2026-02-22T21:40:45.774 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17700 IOs, 7649174528 bytes 2026-02-22T21:40:45.819 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17800 IOs, 7696559104 bytes 2026-02-22T21:40:45.846 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17900 IOs, 7697178624 bytes 2026-02-22T21:40:45.879 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18000 IOs, 7712875520 bytes 2026-02-22T21:40:45.931 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18100 IOs, 7763820544 bytes 2026-02-22T21:40:45.981 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18200 IOs, 7810332672 bytes 2026-02-22T21:40:46.013 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18300 IOs, 7812655104 bytes 2026-02-22T21:40:46.054 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18400 IOs, 7842904064 bytes 2026-02-22T21:40:46.102 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18500 IOs, 7898989568 bytes 2026-02-22T21:40:46.150 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18600 IOs, 7944919040 bytes 2026-02-22T21:40:46.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18700 IOs, 8018997248 bytes 2026-02-22T21:40:46.235 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18800 IOs, 8019612672 bytes 2026-02-22T21:40:46.273 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18900 IOs, 8048004096 bytes 2026-02-22T21:40:46.332 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19000 IOs, 8119786496 bytes 2026-02-22T21:40:46.407 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19100 IOs, 8230259712 bytes 2026-02-22T21:40:46.441 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19200 IOs, 8242566144 bytes 2026-02-22T21:40:46.470 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19300 IOs, 8247994368 bytes 2026-02-22T21:40:46.522 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19400 IOs, 8307054592 bytes 2026-02-22T21:40:46.566 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19500 IOs, 8350041088 bytes 2026-02-22T21:40:46.632 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19600 IOs, 8423124992 bytes 2026-02-22T21:40:46.657 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19700 IOs, 8423719936 bytes 2026-02-22T21:40:46.701 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19800 IOs, 8465642496 bytes 2026-02-22T21:40:46.742 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19900 IOs, 8505387008 bytes 2026-02-22T21:40:46.804 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20000 IOs, 8562023424 bytes 2026-02-22T21:40:46.829 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20100 IOs, 8565123072 bytes 2026-02-22T21:40:46.867 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20200 IOs, 8588675072 bytes 2026-02-22T21:40:46.920 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20300 IOs, 8663304192 bytes 2026-02-22T21:40:46.984 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20400 IOs, 8755529728 bytes 2026-02-22T21:40:47.023 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20500 IOs, 8784603136 bytes 2026-02-22T21:40:47.068 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20600 IOs, 8831467520 bytes 2026-02-22T21:40:47.105 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20700 IOs, 8854999040 bytes 2026-02-22T21:40:47.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20800 IOs, 8936556544 bytes 2026-02-22T21:40:47.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20900 IOs, 8955766784 bytes 2026-02-22T21:40:47.244 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21000 IOs, 8997288960 bytes 2026-02-22T21:40:47.297 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21100 IOs, 9067610112 bytes 2026-02-22T21:40:47.350 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21200 IOs, 9134418944 bytes 2026-02-22T21:40:47.391 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21300 IOs, 9154275328 bytes 2026-02-22T21:40:47.429 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21400 IOs, 9192936448 bytes 2026-02-22T21:40:47.477 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21500 IOs, 9237879808 bytes 2026-02-22T21:40:47.512 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21600 IOs, 9261467648 bytes 2026-02-22T21:40:47.579 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21700 IOs, 9346505728 bytes 2026-02-22T21:40:47.631 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21800 IOs, 9397521408 bytes 2026-02-22T21:40:47.679 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21900 IOs, 9451439104 bytes 2026-02-22T21:40:47.711 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22000 IOs, 9465532416 bytes 2026-02-22T21:40:47.746 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22100 IOs, 9482841088 bytes 2026-02-22T21:40:47.813 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22200 IOs, 9573697536 bytes 2026-02-22T21:40:47.851 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22300 IOs, 9605061632 bytes 2026-02-22T21:40:47.907 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22400 IOs, 9662357504 bytes 2026-02-22T21:40:47.944 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22500 IOs, 9691034624 bytes 2026-02-22T21:40:48.008 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22600 IOs, 9800012800 bytes 2026-02-22T21:40:48.016 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ error_code=124 2026-02-22T21:40:48.016 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set -e 2026-02-22T21:40:48.016 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' 124 -eq 124 ']' 2026-02-22T21:40:48.016 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:40:48.016 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T21:40:48.017 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:40:48.017 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:40:48.017 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:40:48.017 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state_pattern=up+replaying 2026-02-22T21:40:48.017 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local description_pattern= 2026-02-22T21:40:48.017 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local service_pattern= 2026-02-22T21:40:48.017 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:40:48.017 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T21:40:49.017 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T21:40:49.017 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:40:49.018 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:40:49.018 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:40:49.018 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state_pattern=up+replaying 2026-02-22T21:40:49.018 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local description_pattern= 2026-02-22T21:40:49.018 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local service_pattern= 2026-02-22T21:40:49.018 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status 2026-02-22T21:40:49.018 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ CEPH_ARGS= 2026-02-22T21:40:49.018 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:40:49.051 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ status='test: 2026-02-22T21:40:49.051 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: global_id: b410963b-cae9-4337-a6ef-da65321c9374 2026-02-22T21:40:49.051 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: state: up+replaying 2026-02-22T21:40:49.051 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":534851072.0,"last_snapshot_bytes":536848384,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771796378,"remote_snapshot_timestamp":1771796378,"replay_state":"idle"} 2026-02-22T21:40:49.051 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: service: mirror.0 on trial026 2026-02-22T21:40:49.051 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: last_update: 2026-02-22 21:40:32 2026-02-22T21:40:49.051 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: peer_sites: 2026-02-22T21:40:49.051 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: name: cluster2 2026-02-22T21:40:49.051 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: state: up+stopped 2026-02-22T21:40:49.052 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: description: local image is primary 2026-02-22T21:40:49.052 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: last_update: 2026-02-22 21:40:33' 2026-02-22T21:40:49.052 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T21:40:49.052 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: state: up+replaying 2026-02-22T21:40:49.053 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep '^ description: .*' 2026-02-22T21:40:49.053 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":534851072.0,"last_snapshot_bytes":536848384,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771796378,"remote_snapshot_timestamp":1771796378,"replay_state":"idle"} 2026-02-22T21:40:49.053 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n '' ']' 2026-02-22T21:40:49.054 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ echo up+replaying 2026-02-22T21:40:49.054 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep '^up+' 2026-02-22T21:40:49.054 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:up+replaying 2026-02-22T21:40:49.054 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:40:49.055 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: service: mirror.0 on trial026 2026-02-22T21:40:49.055 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local last_update 2026-02-22T21:40:49.055 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:40:49.057 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ last_update='2026-02-22 21:40:32' 2026-02-22T21:40:49.057 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 21:40:32' 2026-02-22T21:40:49.057 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:40:49.058 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:40:49.058 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:40:49.058 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state_pattern=up+replaying 2026-02-22T21:40:49.058 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local 'prev_last_update=2026-02-22 21:40:32' 2026-02-22T21:40:49.058 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status 2026-02-22T21:40:49.058 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ CEPH_ARGS= 2026-02-22T21:40:49.058 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:40:49.088 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ status='OKOKOK142074258mirror.0trial02619.2.3-689-gf90f4eaetrueOK42104261mirror.1trial02619.2.3-689-gf90f4eaefalseOK42184265mirror.3trial02619.2.3-689-gf90f4eaefalseOK43584361mirror.2trial02619.2.3-689-gf90f4eaefalseOKtestb410963b-cae9-4337-a6ef-da65321c9374up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":534851072.0,"last_snapshot_bytes":536848384,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771796378,"remote_snapshot_timestamp":1771796378,"replay_state":"idle"}42074258mirror.0trial0262026-02-22 21:40:32cluster2cfa81703-817a-4074-adbc-5e6e200790bfup+stoppedlocal image is primary2026-02-22 21:40:33' 2026-02-22T21:40:49.089 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local last_update state 2026-02-22T21:40:49.089 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:40:49.090 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ last_update='2026-02-22 21:40:32' 2026-02-22T21:40:49.090 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:40:49.092 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ state=up+replaying 2026-02-22T21:40:49.092 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ echo up+replaying 2026-02-22T21:40:49.092 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep up+replaying 2026-02-22T21:40:49.093 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:up+replaying 2026-02-22T21:40:49.094 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:40:49.094 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:40:49.094 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ snap_name=snap4 2026-02-22T21:40:49.094 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_snap cluster2 mirror test snap4 2026-02-22T21:40:49.094 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:40:49.094 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:40:49.094 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:40:49.094 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap4 2026-02-22T21:40:49.094 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap4 --debug-rbd=20 --debug-journaler=20 2026-02-22T21:41:18.105 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T21:41:18.105 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:41:18.105 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:41:18.105 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:41:18.105 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T21:41:18.106 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:41:18.106 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:41:18.106 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:41:18.106 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=started 2026-02-22T21:41:18.106 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T21:41:18.106 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:41:18.106 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T21:41:19.106 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T21:41:19.106 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:41:19.106 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:41:19.106 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:41:19.106 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=started 2026-02-22T21:41:19.106 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_result 2026-02-22T21:41:19.107 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=stopped 2026-02-22T21:41:19.107 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:41:19.107 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_instance=cluster1 2026-02-22T21:41:19.107 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ grep -i state 2026-02-22T21:41:19.107 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:41:19.107 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T21:41:19.107 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance=cluster1 2026-02-22T21:41:19.107 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local loop_instance 2026-02-22T21:41:19.107 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:41:19.107 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sleep 0 2026-02-22T21:41:19.108 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:41:19.108 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ seq 0 3 2026-02-22T21:41:19.111 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:41:19.111 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:41:19.111 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:0 2026-02-22T21:41:19.112 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:41:19.112 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:41:19.112 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:41:19.112 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:0 2026-02-22T21:41:19.112 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:41:19.112 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:41:19.112 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:41:19.113 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=0 2026-02-22T21:41:19.113 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:41:19.113 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:41:19.113 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:41:19.113 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=0 2026-02-22T21:41:19.113 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T21:41:19.114 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:41:19.114 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:41:19.114 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:41:19.114 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:41:19.114 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:41:19.114 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:0 2026-02-22T21:41:19.114 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:41:19.115 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:41:19.115 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:41:19.115 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T21:41:19.115 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:41:19.115 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:41:19.115 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:41:19.115 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=0 2026-02-22T21:41:19.115 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=0 2026-02-22T21:41:19.115 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:41:19.124 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:41:19.124 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:41:19.124 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:41:19.124 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:41:19.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ return 0 2026-02-22T21:41:19.173 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:41:19.173 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:41:19.173 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep -i Replaying 2026-02-22T21:41:19.173 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: "state": "Replaying" 2026-02-22T21:41:19.175 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_state=started 2026-02-22T21:41:19.175 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test started = started 2026-02-22T21:41:19.175 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:41:19.175 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2026-02-22T21:41:19.175 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T21:41:19.175 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:41:19.175 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:41:19.175 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:41:19.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = journal ']' 2026-02-22T21:41:19.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T21:41:19.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-22T21:41:19.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:41:19.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:41:19.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:41:19.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-22T21:41:21.109 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Snapshot ID: 14 2026-02-22T21:41:21.118 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test 2026-02-22T21:41:21.118 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T21:41:21.118 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:41:21.118 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:41:21.118 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:41:21.118 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster2-mirror-test.status 2026-02-22T21:41:21.118 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster2-mirror-test.status 2026-02-22T21:41:21.119 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T21:41:21.119 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_log=/tmp/tmp.j44LZPowal/cluster2-mirror-test.status 2026-02-22T21:41:21.120 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster1-mirror-test.status 2026-02-22T21:41:21.120 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster1-mirror-test.status 2026-02-22T21:41:21.120 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T21:41:21.121 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_status_log=/tmp/tmp.j44LZPowal/cluster1-mirror-test.status 2026-02-22T21:41:21.121 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.j44LZPowal/cluster2-mirror-test.status 2026-02-22T21:41:21.121 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:41:21.121 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:41:21.121 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:41:21.121 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster2-mirror-test.status 2026-02-22T21:41:21.122 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-22T21:41:21.122 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:41:21.152 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T21:41:21.154 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snapshot_id=14 2026-02-22T21:41:21.155 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:41:21.155 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:41:21.155 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.2 2026-02-22T21:41:21.356 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.j44LZPowal/cluster1-mirror-test.status 2026-02-22T21:41:21.356 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:41:21.356 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:41:21.356 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:41:21.356 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-test.status 2026-02-22T21:41:21.356 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T21:41:21.356 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:41:21.385 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:41:21.385 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:41:21.386 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T21:41:21.386 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T21:41:21.386 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T21:41:21.386 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T21:41:21.386 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 14 = '' 2026-02-22T21:41:21.386 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:41:21.386 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.4 2026-02-22T21:41:21.788 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.j44LZPowal/cluster1-mirror-test.status 2026-02-22T21:41:21.789 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:41:21.789 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:41:21.789 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:41:21.789 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-test.status 2026-02-22T21:41:21.789 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T21:41:21.789 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:41:21.817 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:41:21.818 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:41:21.818 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T21:41:21.819 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T21:41:21.819 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T21:41:21.819 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T21:41:21.819 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 14 = '' 2026-02-22T21:41:21.819 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:41:21.819 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.8 2026-02-22T21:41:22.621 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.j44LZPowal/cluster1-mirror-test.status 2026-02-22T21:41:22.621 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:41:22.621 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:41:22.621 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:41:22.621 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-test.status 2026-02-22T21:41:22.621 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T21:41:22.621 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:41:22.682 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:41:22.682 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:41:22.683 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T21:41:22.684 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T21:41:22.684 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T21:41:22.684 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T21:41:22.685 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 14 = '' 2026-02-22T21:41:22.685 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:41:22.685 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1.6 2026-02-22T21:41:24.286 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.j44LZPowal/cluster1-mirror-test.status 2026-02-22T21:41:24.286 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:41:24.286 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:41:24.286 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:41:24.286 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-test.status 2026-02-22T21:41:24.286 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T21:41:24.286 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:41:24.318 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:41:24.320 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id=14 2026-02-22T21:41:24.320 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 14 = 14 2026-02-22T21:41:24.320 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:41:24.320 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snap_present cluster1 mirror test snap4 2026-02-22T21:41:24.321 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:41:24.321 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:41:24.321 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:41:24.321 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap4 2026-02-22T21:41:24.321 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T21:41:24.321 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T21:41:24.321 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T21:41:25.320 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 info mirror/test@snap4 2026-02-22T21:41:25.347 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'test': 2026-02-22T21:41:25.347 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 512 MiB in 128 objects 2026-02-22T21:41:25.347 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T21:41:25.347 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 8 2026-02-22T21:41:25.347 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 10a25aa14179 2026-02-22T21:41:25.347 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.10a25aa14179 2026-02-22T21:41:25.347 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T21:41:25.347 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock, non-primary 2026-02-22T21:41:25.347 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T21:41:25.347 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T21:41:25.347 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:38:05 2026 2026-02-22T21:41:25.348 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:39:43 2026 2026-02-22T21:41:25.348 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:38:05 2026 2026-02-22T21:41:25.348 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: protected: False 2026-02-22T21:41:25.348 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T21:41:25.348 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T21:41:25.348 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: b410963b-cae9-4337-a6ef-da65321c9374 2026-02-22T21:41:25.348 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: false 2026-02-22T21:41:25.349 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:41:25.349 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n snap3-clean ']' 2026-02-22T21:41:25.349 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ compare_image_snaps mirror test snap3-clean 2026-02-22T21:41:25.349 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:41:25.349 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:41:25.349 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap3-clean 2026-02-22T21:41:25.349 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local ret=0 2026-02-22T21:41:25.349 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local rmt_export=/tmp/tmp.j44LZPowal/cluster2-mirror-test.export 2026-02-22T21:41:25.349 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local loc_export=/tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:41:25.349 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:41:25.349 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror export test@snap3-clean /tmp/tmp.j44LZPowal/cluster2-mirror-test.export 2026-02-22T21:41:25.687 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-22T21:41:25.697 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 -p mirror export test@snap3-clean /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:41:26.072 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-22T21:41:26.083 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ cmp /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:41:26.262 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:41:26.387 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:41:26.388 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ compare_image_snaps mirror test snap4 2026-02-22T21:41:26.388 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:41:26.388 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:41:26.388 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap4 2026-02-22T21:41:26.388 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local ret=0 2026-02-22T21:41:26.388 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local rmt_export=/tmp/tmp.j44LZPowal/cluster2-mirror-test.export 2026-02-22T21:41:26.388 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local loc_export=/tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:41:26.388 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:41:26.389 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror export test@snap4 /tmp/tmp.j44LZPowal/cluster2-mirror-test.export 2026-02-22T21:41:26.930 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-22T21:41:26.940 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 -p mirror export test@snap4 /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:41:27.335 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-22T21:41:27.345 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ cmp /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:41:27.529 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:41:27.657 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:41:27.657 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ clean_snap_name=snap4-clean 2026-02-22T21:41:27.657 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_snap cluster2 mirror test snap4-clean 2026-02-22T21:41:27.657 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:41:27.657 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:41:27.657 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:41:27.657 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap4-clean 2026-02-22T21:41:27.657 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap4-clean --debug-rbd=20 --debug-journaler=20 2026-02-22T21:41:28.117 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 10` 2026-02-22T21:41:28.117 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ stress_write_image cluster2 mirror test 2026-02-22T21:41:28.117 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:41:28.117 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:41:28.117 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:41:28.117 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-22T21:41:28.119 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local duration=12 2026-02-22T21:41:28.119 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set +e 2026-02-22T21:41:28.119 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ timeout 12s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2026-02-22T21:41:28.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:100 IOs, 43222016 bytes 2026-02-22T21:41:28.294 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:200 IOs, 89259008 bytes 2026-02-22T21:41:28.345 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:300 IOs, 156918784 bytes 2026-02-22T21:41:28.383 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:400 IOs, 163749888 bytes 2026-02-22T21:41:28.424 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:500 IOs, 194156544 bytes 2026-02-22T21:41:28.460 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:600 IOs, 233893888 bytes 2026-02-22T21:41:28.493 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:700 IOs, 234492928 bytes 2026-02-22T21:41:28.534 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:800 IOs, 291120128 bytes 2026-02-22T21:41:28.594 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:900 IOs, 350388224 bytes 2026-02-22T21:41:28.625 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1000 IOs, 381063168 bytes 2026-02-22T21:41:28.657 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1100 IOs, 381684736 bytes 2026-02-22T21:41:28.704 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1200 IOs, 437739520 bytes 2026-02-22T21:41:28.758 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1300 IOs, 485724160 bytes 2026-02-22T21:41:28.809 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1400 IOs, 541702144 bytes 2026-02-22T21:41:28.843 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1500 IOs, 554240000 bytes 2026-02-22T21:41:28.878 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1600 IOs, 598873088 bytes 2026-02-22T21:41:28.937 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1700 IOs, 662231040 bytes 2026-02-22T21:41:29.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1800 IOs, 714302464 bytes 2026-02-22T21:41:29.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1900 IOs, 727475200 bytes 2026-02-22T21:41:29.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2000 IOs, 765834240 bytes 2026-02-22T21:41:29.322 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2100 IOs, 868972544 bytes 2026-02-22T21:41:29.358 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2200 IOs, 889589760 bytes 2026-02-22T21:41:29.384 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2300 IOs, 893319168 bytes 2026-02-22T21:41:29.423 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2400 IOs, 924840960 bytes 2026-02-22T21:41:29.476 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2500 IOs, 985404416 bytes 2026-02-22T21:41:29.505 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2600 IOs, 990319616 bytes 2026-02-22T21:41:29.544 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2700 IOs, 1019633664 bytes 2026-02-22T21:41:29.581 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2800 IOs, 1049737216 bytes 2026-02-22T21:41:29.606 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2900 IOs, 1050352640 bytes 2026-02-22T21:41:29.644 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3000 IOs, 1078988800 bytes 2026-02-22T21:41:29.699 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3100 IOs, 1151084544 bytes 2026-02-22T21:41:29.748 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3200 IOs, 1217811456 bytes 2026-02-22T21:41:29.800 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3300 IOs, 1274351616 bytes 2026-02-22T21:41:29.867 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3400 IOs, 1353834496 bytes 2026-02-22T21:41:29.925 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3500 IOs, 1419029504 bytes 2026-02-22T21:41:29.976 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3600 IOs, 1466657792 bytes 2026-02-22T21:41:30.021 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3700 IOs, 1526923264 bytes 2026-02-22T21:41:30.076 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3800 IOs, 1591540736 bytes 2026-02-22T21:41:30.118 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3900 IOs, 1623490560 bytes 2026-02-22T21:41:30.154 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4000 IOs, 1651226624 bytes 2026-02-22T21:41:30.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4100 IOs, 1736474624 bytes 2026-02-22T21:41:30.279 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4200 IOs, 1831188480 bytes 2026-02-22T21:41:30.312 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4300 IOs, 1838766080 bytes 2026-02-22T21:41:30.355 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4400 IOs, 1882044416 bytes 2026-02-22T21:41:30.393 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4500 IOs, 1911049216 bytes 2026-02-22T21:41:30.442 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4600 IOs, 1949834240 bytes 2026-02-22T21:41:30.502 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4700 IOs, 2027543552 bytes 2026-02-22T21:41:30.532 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4800 IOs, 2034701312 bytes 2026-02-22T21:41:30.563 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4900 IOs, 2052997120 bytes 2026-02-22T21:41:30.618 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5000 IOs, 2106792960 bytes 2026-02-22T21:41:30.665 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5100 IOs, 2152397824 bytes 2026-02-22T21:41:30.719 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5200 IOs, 2217595904 bytes 2026-02-22T21:41:30.751 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5300 IOs, 2230111232 bytes 2026-02-22T21:41:30.794 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5400 IOs, 2269800448 bytes 2026-02-22T21:41:30.835 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5500 IOs, 2303160320 bytes 2026-02-22T21:41:30.883 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5600 IOs, 2355358720 bytes 2026-02-22T21:41:30.946 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5700 IOs, 2438533120 bytes 2026-02-22T21:41:30.993 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5800 IOs, 2503997440 bytes 2026-02-22T21:41:31.048 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5900 IOs, 2561473536 bytes 2026-02-22T21:41:31.079 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6000 IOs, 2568949760 bytes 2026-02-22T21:41:31.112 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6100 IOs, 2585336832 bytes 2026-02-22T21:41:31.157 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6200 IOs, 2643723264 bytes 2026-02-22T21:41:31.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6300 IOs, 2683137024 bytes 2026-02-22T21:41:31.257 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6400 IOs, 2781469696 bytes 2026-02-22T21:41:31.328 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6500 IOs, 2871158784 bytes 2026-02-22T21:41:31.362 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6600 IOs, 2892021760 bytes 2026-02-22T21:41:31.387 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6700 IOs, 2895429632 bytes 2026-02-22T21:41:31.441 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6800 IOs, 2954467328 bytes 2026-02-22T21:41:31.477 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6900 IOs, 2985748480 bytes 2026-02-22T21:41:31.536 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7000 IOs, 3058976768 bytes 2026-02-22T21:41:31.576 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7100 IOs, 3104520192 bytes 2026-02-22T21:41:31.610 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7200 IOs, 3108511744 bytes 2026-02-22T21:41:31.657 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7300 IOs, 3154493440 bytes 2026-02-22T21:41:31.691 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7400 IOs, 3172024320 bytes 2026-02-22T21:41:31.724 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7500 IOs, 3203536896 bytes 2026-02-22T21:41:31.778 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7600 IOs, 3264309248 bytes 2026-02-22T21:41:31.844 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7700 IOs, 3355727872 bytes 2026-02-22T21:41:31.884 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7800 IOs, 3413989376 bytes 2026-02-22T21:41:31.930 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7900 IOs, 3438056448 bytes 2026-02-22T21:41:31.972 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8000 IOs, 3476293632 bytes 2026-02-22T21:41:32.016 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8100 IOs, 3528070144 bytes 2026-02-22T21:41:32.079 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8200 IOs, 3605770240 bytes 2026-02-22T21:41:32.119 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8300 IOs, 3650255872 bytes 2026-02-22T21:41:32.153 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8400 IOs, 3650878464 bytes 2026-02-22T21:41:32.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8500 IOs, 3717728256 bytes 2026-02-22T21:41:32.257 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8600 IOs, 3775311872 bytes 2026-02-22T21:41:32.290 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8700 IOs, 3800214528 bytes 2026-02-22T21:41:32.339 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8800 IOs, 3857409024 bytes 2026-02-22T21:41:32.382 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8900 IOs, 3893625856 bytes 2026-02-22T21:41:32.411 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9000 IOs, 3914719232 bytes 2026-02-22T21:41:32.453 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9100 IOs, 3937783808 bytes 2026-02-22T21:41:32.505 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9200 IOs, 3998882816 bytes 2026-02-22T21:41:32.546 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9300 IOs, 4025396224 bytes 2026-02-22T21:41:32.599 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9400 IOs, 4077352960 bytes 2026-02-22T21:41:32.649 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9500 IOs, 4114338816 bytes 2026-02-22T21:41:32.674 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9600 IOs, 4114937856 bytes 2026-02-22T21:41:32.713 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9700 IOs, 4141998080 bytes 2026-02-22T21:41:32.758 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9800 IOs, 4180491264 bytes 2026-02-22T21:41:32.819 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9900 IOs, 4268336128 bytes 2026-02-22T21:41:32.881 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10000 IOs, 4367300608 bytes 2026-02-22T21:41:32.923 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10100 IOs, 4399997952 bytes 2026-02-22T21:41:32.970 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10200 IOs, 4454730752 bytes 2026-02-22T21:41:32.991 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.4837136530085516) > weight (0.25) 2026-02-22T21:41:32.991 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.9335632102269479) > weight (0.25) 2026-02-22T21:41:32.991 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.8068690589072798) > weight (0.25) 2026-02-22T21:41:32.991 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.3 with skip (0.9664097668336108) > weight (0.25) 2026-02-22T21:41:32.991 INFO:tasks.rbd_mirror_thrash:waiting for 67.16652069233372 secs before thrashing 2026-02-22T21:41:33.004 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10300 IOs, 4472836096 bytes 2026-02-22T21:41:33.046 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10400 IOs, 4508453888 bytes 2026-02-22T21:41:33.087 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10500 IOs, 4550328320 bytes 2026-02-22T21:41:33.128 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10600 IOs, 4583980032 bytes 2026-02-22T21:41:33.173 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10700 IOs, 4623767552 bytes 2026-02-22T21:41:33.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10800 IOs, 4663694336 bytes 2026-02-22T21:41:33.267 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10900 IOs, 4719007744 bytes 2026-02-22T21:41:33.317 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11000 IOs, 4771451904 bytes 2026-02-22T21:41:33.374 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11100 IOs, 4831102976 bytes 2026-02-22T21:41:33.635 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11200 IOs, 4904962048 bytes 2026-02-22T21:41:33.677 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11300 IOs, 4953227264 bytes 2026-02-22T21:41:33.738 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11400 IOs, 5019902976 bytes 2026-02-22T21:41:33.783 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11500 IOs, 5067083776 bytes 2026-02-22T21:41:33.816 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11600 IOs, 5074859008 bytes 2026-02-22T21:41:33.872 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11700 IOs, 5153949696 bytes 2026-02-22T21:41:33.935 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11800 IOs, 5234004992 bytes 2026-02-22T21:41:33.983 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11900 IOs, 5280275456 bytes 2026-02-22T21:41:34.023 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12000 IOs, 5316670464 bytes 2026-02-22T21:41:34.062 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12100 IOs, 5351095296 bytes 2026-02-22T21:41:34.110 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12200 IOs, 5401431040 bytes 2026-02-22T21:41:34.155 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12300 IOs, 5439364096 bytes 2026-02-22T21:41:34.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12400 IOs, 5442278400 bytes 2026-02-22T21:41:34.218 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12500 IOs, 5467723776 bytes 2026-02-22T21:41:34.261 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12600 IOs, 5522126848 bytes 2026-02-22T21:41:34.316 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12700 IOs, 5575998464 bytes 2026-02-22T21:41:34.361 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12800 IOs, 5643363328 bytes 2026-02-22T21:41:34.418 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12900 IOs, 5684518912 bytes 2026-02-22T21:41:34.465 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13000 IOs, 5729490944 bytes 2026-02-22T21:41:34.512 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13100 IOs, 5780822016 bytes 2026-02-22T21:41:34.556 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13200 IOs, 5813920768 bytes 2026-02-22T21:41:34.608 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13300 IOs, 5873252352 bytes 2026-02-22T21:41:34.646 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13400 IOs, 5886039040 bytes 2026-02-22T21:41:34.686 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13500 IOs, 5929758720 bytes 2026-02-22T21:41:34.728 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13600 IOs, 5955825664 bytes 2026-02-22T21:41:34.766 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13700 IOs, 5980797952 bytes 2026-02-22T21:41:34.791 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13800 IOs, 5981435904 bytes 2026-02-22T21:41:34.850 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13900 IOs, 6055846912 bytes 2026-02-22T21:41:34.881 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14000 IOs, 6075737088 bytes 2026-02-22T21:41:34.940 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14100 IOs, 6148752384 bytes 2026-02-22T21:41:34.971 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14200 IOs, 6180674560 bytes 2026-02-22T21:41:35.021 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14300 IOs, 6220844032 bytes 2026-02-22T21:41:35.064 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14400 IOs, 6260220928 bytes 2026-02-22T21:41:35.101 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14500 IOs, 6282662912 bytes 2026-02-22T21:41:35.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14600 IOs, 6363363328 bytes 2026-02-22T21:41:35.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14700 IOs, 6404685824 bytes 2026-02-22T21:41:35.257 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14800 IOs, 6444945408 bytes 2026-02-22T21:41:35.289 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14900 IOs, 6464382976 bytes 2026-02-22T21:41:35.336 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15000 IOs, 6516370432 bytes 2026-02-22T21:41:35.361 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15100 IOs, 6517003264 bytes 2026-02-22T21:41:35.425 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15200 IOs, 6587343872 bytes 2026-02-22T21:41:35.462 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15300 IOs, 6610669568 bytes 2026-02-22T21:41:35.503 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15400 IOs, 6640152576 bytes 2026-02-22T21:41:35.564 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15500 IOs, 6725795840 bytes 2026-02-22T21:41:35.619 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15600 IOs, 6786019328 bytes 2026-02-22T21:41:35.672 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15700 IOs, 6843191296 bytes 2026-02-22T21:41:35.717 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15800 IOs, 6888784896 bytes 2026-02-22T21:41:35.767 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15900 IOs, 6963968000 bytes 2026-02-22T21:41:35.815 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16000 IOs, 6992006144 bytes 2026-02-22T21:41:35.858 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16100 IOs, 7031127040 bytes 2026-02-22T21:41:35.891 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16200 IOs, 7053585408 bytes 2026-02-22T21:41:35.928 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16300 IOs, 7076032512 bytes 2026-02-22T21:41:35.966 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16400 IOs, 7101328384 bytes 2026-02-22T21:41:35.995 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16500 IOs, 7110583296 bytes 2026-02-22T21:41:36.027 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16600 IOs, 7121153024 bytes 2026-02-22T21:41:36.066 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16700 IOs, 7149578240 bytes 2026-02-22T21:41:36.098 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16800 IOs, 7170286592 bytes 2026-02-22T21:41:36.136 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16900 IOs, 7193333760 bytes 2026-02-22T21:41:36.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17000 IOs, 7212343296 bytes 2026-02-22T21:41:36.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17100 IOs, 7229877248 bytes 2026-02-22T21:41:36.241 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17200 IOs, 7257962496 bytes 2026-02-22T21:41:36.269 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17300 IOs, 7270586368 bytes 2026-02-22T21:41:36.310 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17400 IOs, 7310486528 bytes 2026-02-22T21:41:36.369 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17500 IOs, 7389053952 bytes 2026-02-22T21:41:36.410 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17600 IOs, 7438816256 bytes 2026-02-22T21:41:36.478 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17700 IOs, 7515055104 bytes 2026-02-22T21:41:36.519 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17800 IOs, 7558440960 bytes 2026-02-22T21:41:36.564 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17900 IOs, 7604792320 bytes 2026-02-22T21:41:36.600 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18000 IOs, 7640610816 bytes 2026-02-22T21:41:36.653 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18100 IOs, 7686466560 bytes 2026-02-22T21:41:36.716 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18200 IOs, 7773962240 bytes 2026-02-22T21:41:36.761 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18300 IOs, 7822554112 bytes 2026-02-22T21:41:36.800 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18400 IOs, 7851236352 bytes 2026-02-22T21:41:36.847 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18500 IOs, 7904143360 bytes 2026-02-22T21:41:36.904 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18600 IOs, 7962909696 bytes 2026-02-22T21:41:36.931 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18700 IOs, 7964041216 bytes 2026-02-22T21:41:36.987 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18800 IOs, 8043036672 bytes 2026-02-22T21:41:37.039 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18900 IOs, 8093710336 bytes 2026-02-22T21:41:37.072 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19000 IOs, 8113168384 bytes 2026-02-22T21:41:37.136 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19100 IOs, 8189498368 bytes 2026-02-22T21:41:37.184 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19200 IOs, 8242275328 bytes 2026-02-22T21:41:37.226 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19300 IOs, 8276902912 bytes 2026-02-22T21:41:37.262 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19400 IOs, 8297963520 bytes 2026-02-22T21:41:37.302 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19500 IOs, 8328075264 bytes 2026-02-22T21:41:37.340 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19600 IOs, 8357862400 bytes 2026-02-22T21:41:37.381 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19700 IOs, 8395599872 bytes 2026-02-22T21:41:37.409 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19800 IOs, 8407982080 bytes 2026-02-22T21:41:37.461 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19900 IOs, 8456806400 bytes 2026-02-22T21:41:37.508 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20000 IOs, 8551212032 bytes 2026-02-22T21:41:37.569 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20100 IOs, 8577672192 bytes 2026-02-22T21:41:37.620 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20200 IOs, 8642288640 bytes 2026-02-22T21:41:37.650 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20300 IOs, 8646875136 bytes 2026-02-22T21:41:37.684 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20400 IOs, 8671166464 bytes 2026-02-22T21:41:37.740 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20500 IOs, 8720868352 bytes 2026-02-22T21:41:37.775 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20600 IOs, 8740827136 bytes 2026-02-22T21:41:37.807 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20700 IOs, 8744835072 bytes 2026-02-22T21:41:37.836 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20800 IOs, 8756806656 bytes 2026-02-22T21:41:37.878 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20900 IOs, 8782089216 bytes 2026-02-22T21:41:37.914 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21000 IOs, 8809790464 bytes 2026-02-22T21:41:37.962 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21100 IOs, 8865580032 bytes 2026-02-22T21:41:38.023 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21200 IOs, 8936483840 bytes 2026-02-22T21:41:38.101 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21300 IOs, 9049348096 bytes 2026-02-22T21:41:38.138 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21400 IOs, 9067294720 bytes 2026-02-22T21:41:38.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21500 IOs, 9134932992 bytes 2026-02-22T21:41:38.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21600 IOs, 9203999744 bytes 2026-02-22T21:41:38.295 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21700 IOs, 9233957888 bytes 2026-02-22T21:41:38.333 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21800 IOs, 9258955776 bytes 2026-02-22T21:41:38.370 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21900 IOs, 9281546240 bytes 2026-02-22T21:41:38.404 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22000 IOs, 9304676352 bytes 2026-02-22T21:41:38.484 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22100 IOs, 9420457984 bytes 2026-02-22T21:41:38.520 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22200 IOs, 9446118400 bytes 2026-02-22T21:41:38.580 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22300 IOs, 9515525120 bytes 2026-02-22T21:41:38.615 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22400 IOs, 9542428672 bytes 2026-02-22T21:41:38.680 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22500 IOs, 9613717504 bytes 2026-02-22T21:41:38.730 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22600 IOs, 9667962880 bytes 2026-02-22T21:41:38.766 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22700 IOs, 9695293440 bytes 2026-02-22T21:41:38.815 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22800 IOs, 9751753728 bytes 2026-02-22T21:41:38.873 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22900 IOs, 9822822400 bytes 2026-02-22T21:41:38.902 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23000 IOs, 9825122304 bytes 2026-02-22T21:41:38.942 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23100 IOs, 9880091648 bytes 2026-02-22T21:41:38.979 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23200 IOs, 9880704000 bytes 2026-02-22T21:41:39.027 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23300 IOs, 9931582464 bytes 2026-02-22T21:41:39.103 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23400 IOs, 10042225664 bytes 2026-02-22T21:41:39.150 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23500 IOs, 10090931200 bytes 2026-02-22T21:41:39.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23600 IOs, 10119275520 bytes 2026-02-22T21:41:39.234 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23700 IOs, 10157764608 bytes 2026-02-22T21:41:39.264 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23800 IOs, 10166760448 bytes 2026-02-22T21:41:39.311 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23900 IOs, 10219522048 bytes 2026-02-22T21:41:39.359 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24000 IOs, 10268411904 bytes 2026-02-22T21:41:39.409 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24100 IOs, 10329592832 bytes 2026-02-22T21:41:39.450 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24200 IOs, 10353137664 bytes 2026-02-22T21:41:39.489 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24300 IOs, 10381146112 bytes 2026-02-22T21:41:39.536 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24400 IOs, 10428924928 bytes 2026-02-22T21:41:39.594 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24500 IOs, 10516290560 bytes 2026-02-22T21:41:39.646 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24600 IOs, 10562442240 bytes 2026-02-22T21:41:39.693 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24700 IOs, 10601080832 bytes 2026-02-22T21:41:39.742 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24800 IOs, 10641332224 bytes 2026-02-22T21:41:39.783 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24900 IOs, 10667230208 bytes 2026-02-22T21:41:39.833 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25000 IOs, 10727165952 bytes 2026-02-22T21:41:39.890 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25100 IOs, 10767696896 bytes 2026-02-22T21:41:39.946 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25200 IOs, 10841495552 bytes 2026-02-22T21:41:39.996 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25300 IOs, 10865213440 bytes 2026-02-22T21:41:40.063 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25400 IOs, 10932188160 bytes 2026-02-22T21:41:40.125 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ error_code=124 2026-02-22T21:41:40.125 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set -e 2026-02-22T21:41:40.125 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' 124 -eq 124 ']' 2026-02-22T21:41:40.125 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:41:40.125 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T21:41:40.125 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:41:40.126 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:41:40.126 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:41:40.126 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state_pattern=up+replaying 2026-02-22T21:41:40.126 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local description_pattern= 2026-02-22T21:41:40.126 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local service_pattern= 2026-02-22T21:41:40.126 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:41:40.126 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T21:41:41.125 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T21:41:41.125 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:41:41.125 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:41:41.125 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:41:41.125 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state_pattern=up+replaying 2026-02-22T21:41:41.125 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local description_pattern= 2026-02-22T21:41:41.125 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local service_pattern= 2026-02-22T21:41:41.125 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status 2026-02-22T21:41:41.125 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ CEPH_ARGS= 2026-02-22T21:41:41.125 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:41:41.158 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ status='test: 2026-02-22T21:41:41.158 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: global_id: b410963b-cae9-4337-a6ef-da65321c9374 2026-02-22T21:41:41.158 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: state: up+replaying 2026-02-22T21:41:41.158 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: description: replaying, {"bytes_per_second":17880678.4,"bytes_per_snapshot":536441856.0,"last_snapshot_bytes":536420352,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771796480,"remote_snapshot_timestamp":1771796480,"replay_state":"idle"} 2026-02-22T21:41:41.158 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: service: mirror.0 on trial026 2026-02-22T21:41:41.158 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: last_update: 2026-02-22 21:41:32 2026-02-22T21:41:41.158 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: peer_sites: 2026-02-22T21:41:41.158 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: name: cluster2 2026-02-22T21:41:41.158 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: state: up+stopped 2026-02-22T21:41:41.158 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: description: local image is primary 2026-02-22T21:41:41.159 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: last_update: 2026-02-22 21:41:33' 2026-02-22T21:41:41.159 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T21:41:41.159 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: state: up+replaying 2026-02-22T21:41:41.159 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep '^ description: .*' 2026-02-22T21:41:41.160 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: description: replaying, {"bytes_per_second":17880678.4,"bytes_per_snapshot":536441856.0,"last_snapshot_bytes":536420352,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771796480,"remote_snapshot_timestamp":1771796480,"replay_state":"idle"} 2026-02-22T21:41:41.160 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n '' ']' 2026-02-22T21:41:41.160 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ echo up+replaying 2026-02-22T21:41:41.160 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep '^up+' 2026-02-22T21:41:41.161 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:up+replaying 2026-02-22T21:41:41.161 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:41:41.162 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: service: mirror.0 on trial026 2026-02-22T21:41:41.162 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local last_update 2026-02-22T21:41:41.162 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:41:41.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ last_update='2026-02-22 21:41:32' 2026-02-22T21:41:41.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 21:41:32' 2026-02-22T21:41:41.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:41:41.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:41:41.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:41:41.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state_pattern=up+replaying 2026-02-22T21:41:41.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local 'prev_last_update=2026-02-22 21:41:32' 2026-02-22T21:41:41.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status 2026-02-22T21:41:41.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ CEPH_ARGS= 2026-02-22T21:41:41.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:41:41.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ status='OKOKOK142074258mirror.0trial02619.2.3-689-gf90f4eaetrueOK42104261mirror.1trial02619.2.3-689-gf90f4eaefalseOK42184265mirror.3trial02619.2.3-689-gf90f4eaefalseOK43584361mirror.2trial02619.2.3-689-gf90f4eaefalseOKtestb410963b-cae9-4337-a6ef-da65321c9374up+replayingreplaying, {"bytes_per_second":17880678.4,"bytes_per_snapshot":536441856.0,"last_snapshot_bytes":536420352,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771796480,"remote_snapshot_timestamp":1771796480,"replay_state":"idle"}42074258mirror.0trial0262026-02-22 21:41:32cluster2cfa81703-817a-4074-adbc-5e6e200790bfup+stoppedlocal image is primary2026-02-22 21:41:33' 2026-02-22T21:41:41.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local last_update state 2026-02-22T21:41:41.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:41:41.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ last_update='2026-02-22 21:41:32' 2026-02-22T21:41:41.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:41:41.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ state=up+replaying 2026-02-22T21:41:41.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ echo up+replaying 2026-02-22T21:41:41.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep up+replaying 2026-02-22T21:41:41.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:up+replaying 2026-02-22T21:41:41.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:41:41.203 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:41:41.203 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ snap_name=snap5 2026-02-22T21:41:41.203 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_snap cluster2 mirror test snap5 2026-02-22T21:41:41.203 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:41:41.203 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:41:41.203 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:41:41.203 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap5 2026-02-22T21:41:41.203 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap5 --debug-rbd=20 --debug-journaler=20 2026-02-22T21:42:12.034 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T21:42:12.034 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:42:12.034 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:42:12.034 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:42:12.034 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T21:42:12.034 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:42:12.035 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:42:12.035 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:42:12.035 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=started 2026-02-22T21:42:12.035 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T21:42:12.035 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:42:12.035 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T21:42:13.034 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T21:42:13.034 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:42:13.034 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:42:13.034 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:42:13.034 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=started 2026-02-22T21:42:13.034 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_result 2026-02-22T21:42:13.034 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=stopped 2026-02-22T21:42:13.035 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:42:13.035 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_instance=cluster1 2026-02-22T21:42:13.035 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:42:13.035 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T21:42:13.035 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance=cluster1 2026-02-22T21:42:13.035 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local loop_instance 2026-02-22T21:42:13.035 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:42:13.035 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sleep 0 2026-02-22T21:42:13.035 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ grep -i state 2026-02-22T21:42:13.035 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:42:13.035 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ seq 0 3 2026-02-22T21:42:13.036 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:42:13.036 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:42:13.036 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:0 2026-02-22T21:42:13.036 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:42:13.036 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:42:13.036 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:42:13.036 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:0 2026-02-22T21:42:13.036 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:42:13.036 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:42:13.036 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:42:13.036 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=0 2026-02-22T21:42:13.037 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:42:13.037 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:42:13.037 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:42:13.037 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=0 2026-02-22T21:42:13.037 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T21:42:13.037 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:42:13.037 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:42:13.037 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:42:13.037 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:42:13.037 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:42:13.037 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:0 2026-02-22T21:42:13.037 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:42:13.037 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:42:13.037 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:42:13.037 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T21:42:13.037 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:42:13.037 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:42:13.037 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:42:13.037 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=0 2026-02-22T21:42:13.037 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=0 2026-02-22T21:42:13.037 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:42:13.049 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:42:13.049 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:42:13.049 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:42:13.049 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:42:13.095 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ return 0 2026-02-22T21:42:13.095 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:42:13.095 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:42:13.095 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep -i Replaying 2026-02-22T21:42:13.096 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: "state": "Replaying" 2026-02-22T21:42:13.097 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_state=started 2026-02-22T21:42:13.097 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test started = started 2026-02-22T21:42:13.097 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:42:13.097 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2026-02-22T21:42:13.097 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T21:42:13.097 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:42:13.097 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:42:13.097 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:42:13.097 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = journal ']' 2026-02-22T21:42:13.097 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T21:42:13.097 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-22T21:42:13.098 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:42:13.098 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:42:13.098 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:42:13.098 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-22T21:42:15.038 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Snapshot ID: 17 2026-02-22T21:42:15.047 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test 2026-02-22T21:42:15.047 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T21:42:15.047 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:42:15.047 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:42:15.047 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:42:15.047 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster2-mirror-test.status 2026-02-22T21:42:15.047 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster2-mirror-test.status 2026-02-22T21:42:15.048 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T21:42:15.049 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_log=/tmp/tmp.j44LZPowal/cluster2-mirror-test.status 2026-02-22T21:42:15.049 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster1-mirror-test.status 2026-02-22T21:42:15.049 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster1-mirror-test.status 2026-02-22T21:42:15.049 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T21:42:15.050 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_status_log=/tmp/tmp.j44LZPowal/cluster1-mirror-test.status 2026-02-22T21:42:15.050 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.j44LZPowal/cluster2-mirror-test.status 2026-02-22T21:42:15.050 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:42:15.050 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:42:15.051 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:42:15.051 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster2-mirror-test.status 2026-02-22T21:42:15.051 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-22T21:42:15.051 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:42:15.081 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T21:42:15.083 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snapshot_id=17 2026-02-22T21:42:15.083 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:42:15.083 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:42:15.083 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.2 2026-02-22T21:42:15.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.j44LZPowal/cluster1-mirror-test.status 2026-02-22T21:42:15.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:42:15.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:42:15.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:42:15.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-test.status 2026-02-22T21:42:15.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T21:42:15.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:42:15.314 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:42:15.314 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:42:15.315 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T21:42:15.315 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T21:42:15.315 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T21:42:15.316 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T21:42:15.316 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 17 = '' 2026-02-22T21:42:15.316 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:42:15.316 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.4 2026-02-22T21:42:15.716 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.j44LZPowal/cluster1-mirror-test.status 2026-02-22T21:42:15.716 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:42:15.716 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:42:15.716 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:42:15.716 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-test.status 2026-02-22T21:42:15.717 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T21:42:15.717 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:42:15.747 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:42:15.747 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:42:15.748 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T21:42:15.748 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T21:42:15.748 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T21:42:15.748 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T21:42:15.748 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 17 = '' 2026-02-22T21:42:15.748 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:42:15.749 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.8 2026-02-22T21:42:16.550 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.j44LZPowal/cluster1-mirror-test.status 2026-02-22T21:42:16.550 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:42:16.550 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:42:16.550 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:42:16.550 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-test.status 2026-02-22T21:42:16.550 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T21:42:16.550 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:42:16.595 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:42:16.595 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:42:16.596 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T21:42:16.596 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T21:42:16.596 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T21:42:16.597 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T21:42:16.597 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 17 = '' 2026-02-22T21:42:16.597 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:42:16.597 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1.6 2026-02-22T21:42:18.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.j44LZPowal/cluster1-mirror-test.status 2026-02-22T21:42:18.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:42:18.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:42:18.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:42:18.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-test.status 2026-02-22T21:42:18.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T21:42:18.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:42:18.230 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:42:18.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id=17 2026-02-22T21:42:18.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 17 = 17 2026-02-22T21:42:18.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:42:18.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snap_present cluster1 mirror test snap5 2026-02-22T21:42:18.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:42:18.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:42:18.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:42:18.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap5 2026-02-22T21:42:18.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T21:42:18.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T21:42:18.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T21:42:19.233 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 info mirror/test@snap5 2026-02-22T21:42:19.260 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'test': 2026-02-22T21:42:19.260 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 512 MiB in 128 objects 2026-02-22T21:42:19.261 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T21:42:19.261 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 10 2026-02-22T21:42:19.261 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 10a25aa14179 2026-02-22T21:42:19.261 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.10a25aa14179 2026-02-22T21:42:19.261 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T21:42:19.261 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock, non-primary 2026-02-22T21:42:19.261 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T21:42:19.261 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T21:42:19.261 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:38:05 2026 2026-02-22T21:42:19.261 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:41:25 2026 2026-02-22T21:42:19.262 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:38:05 2026 2026-02-22T21:42:19.262 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: protected: False 2026-02-22T21:42:19.262 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T21:42:19.262 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T21:42:19.262 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: b410963b-cae9-4337-a6ef-da65321c9374 2026-02-22T21:42:19.262 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: false 2026-02-22T21:42:19.263 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:42:19.263 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n snap4-clean ']' 2026-02-22T21:42:19.264 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ compare_image_snaps mirror test snap4-clean 2026-02-22T21:42:19.264 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:42:19.264 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:42:19.264 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap4-clean 2026-02-22T21:42:19.264 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local ret=0 2026-02-22T21:42:19.264 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local rmt_export=/tmp/tmp.j44LZPowal/cluster2-mirror-test.export 2026-02-22T21:42:19.264 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local loc_export=/tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:42:19.264 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:42:19.264 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror export test@snap4-clean /tmp/tmp.j44LZPowal/cluster2-mirror-test.export 2026-02-22T21:42:19.597 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-22T21:42:19.609 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 -p mirror export test@snap4-clean /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:42:19.989 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-22T21:42:20.002 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ cmp /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:42:20.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:42:20.306 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:42:20.306 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ compare_image_snaps mirror test snap5 2026-02-22T21:42:20.306 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:42:20.306 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:42:20.306 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap5 2026-02-22T21:42:20.307 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local ret=0 2026-02-22T21:42:20.307 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local rmt_export=/tmp/tmp.j44LZPowal/cluster2-mirror-test.export 2026-02-22T21:42:20.307 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local loc_export=/tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:42:20.307 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:42:20.307 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror export test@snap5 /tmp/tmp.j44LZPowal/cluster2-mirror-test.export 2026-02-22T21:42:20.650 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-22T21:42:20.661 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 -p mirror export test@snap5 /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:42:21.276 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-22T21:42:21.290 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ cmp /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:42:21.474 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:42:21.600 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:42:21.600 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ clean_snap_name=snap5-clean 2026-02-22T21:42:21.600 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_snap cluster2 mirror test snap5-clean 2026-02-22T21:42:21.601 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:42:21.601 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:42:21.601 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:42:21.601 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap5-clean 2026-02-22T21:42:21.601 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap5-clean --debug-rbd=20 --debug-journaler=20 2026-02-22T21:42:22.048 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 10` 2026-02-22T21:42:22.048 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ stress_write_image cluster2 mirror test 2026-02-22T21:42:22.048 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:42:22.049 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:42:22.049 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:42:22.049 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-22T21:42:22.050 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local duration=13 2026-02-22T21:42:22.050 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set +e 2026-02-22T21:42:22.050 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ timeout 13s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2026-02-22T21:42:22.155 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:100 IOs, 55734272 bytes 2026-02-22T21:42:22.241 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:200 IOs, 119638016 bytes 2026-02-22T21:42:22.296 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:300 IOs, 172017664 bytes 2026-02-22T21:42:22.335 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:400 IOs, 207569920 bytes 2026-02-22T21:42:22.378 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:500 IOs, 237297664 bytes 2026-02-22T21:42:22.410 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:600 IOs, 248556544 bytes 2026-02-22T21:42:22.456 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:700 IOs, 294077440 bytes 2026-02-22T21:42:22.499 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:800 IOs, 348328960 bytes 2026-02-22T21:42:22.541 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:900 IOs, 353757184 bytes 2026-02-22T21:42:22.573 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1000 IOs, 382951424 bytes 2026-02-22T21:42:22.614 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1100 IOs, 401539072 bytes 2026-02-22T21:42:22.648 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1200 IOs, 415873024 bytes 2026-02-22T21:42:22.697 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1300 IOs, 465336320 bytes 2026-02-22T21:42:22.768 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1400 IOs, 566033408 bytes 2026-02-22T21:42:22.833 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1500 IOs, 665411584 bytes 2026-02-22T21:42:22.861 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1600 IOs, 666031104 bytes 2026-02-22T21:42:22.895 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1700 IOs, 688862208 bytes 2026-02-22T21:42:22.959 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1800 IOs, 766289920 bytes 2026-02-22T21:42:22.995 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1900 IOs, 796308480 bytes 2026-02-22T21:42:23.058 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2000 IOs, 870483968 bytes 2026-02-22T21:42:23.102 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2100 IOs, 906553344 bytes 2026-02-22T21:42:23.157 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2200 IOs, 969835520 bytes 2026-02-22T21:42:23.225 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2300 IOs, 1062077440 bytes 2026-02-22T21:42:23.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2400 IOs, 1101693952 bytes 2026-02-22T21:42:23.300 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2500 IOs, 1110162432 bytes 2026-02-22T21:42:23.346 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2600 IOs, 1153511424 bytes 2026-02-22T21:42:23.400 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2700 IOs, 1213436928 bytes 2026-02-22T21:42:23.454 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2800 IOs, 1277621248 bytes 2026-02-22T21:42:23.488 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2900 IOs, 1279055872 bytes 2026-02-22T21:42:23.519 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3000 IOs, 1296256000 bytes 2026-02-22T21:42:23.560 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3100 IOs, 1323362304 bytes 2026-02-22T21:42:23.616 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3200 IOs, 1391623168 bytes 2026-02-22T21:42:23.662 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3300 IOs, 1436023808 bytes 2026-02-22T21:42:23.712 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3400 IOs, 1479734272 bytes 2026-02-22T21:42:23.783 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3500 IOs, 1596717056 bytes 2026-02-22T21:42:23.850 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3600 IOs, 1672395776 bytes 2026-02-22T21:42:23.913 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3700 IOs, 1752010752 bytes 2026-02-22T21:42:23.964 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3800 IOs, 1806305280 bytes 2026-02-22T21:42:23.990 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3900 IOs, 1807354880 bytes 2026-02-22T21:42:24.036 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4000 IOs, 1863016448 bytes 2026-02-22T21:42:24.097 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4100 IOs, 1925024768 bytes 2026-02-22T21:42:24.135 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4200 IOs, 1952060416 bytes 2026-02-22T21:42:24.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4300 IOs, 2042087424 bytes 2026-02-22T21:42:24.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4400 IOs, 2092253184 bytes 2026-02-22T21:42:24.298 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4500 IOs, 2145424384 bytes 2026-02-22T21:42:24.335 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4600 IOs, 2168909824 bytes 2026-02-22T21:42:24.373 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4700 IOs, 2193869824 bytes 2026-02-22T21:42:24.400 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4800 IOs, 2196960256 bytes 2026-02-22T21:42:24.483 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4900 IOs, 2328376320 bytes 2026-02-22T21:42:24.522 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5000 IOs, 2356742144 bytes 2026-02-22T21:42:24.549 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5100 IOs, 2359724032 bytes 2026-02-22T21:42:24.582 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5200 IOs, 2374202368 bytes 2026-02-22T21:42:24.627 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5300 IOs, 2412357632 bytes 2026-02-22T21:42:24.665 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5400 IOs, 2431977472 bytes 2026-02-22T21:42:24.712 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5500 IOs, 2472643584 bytes 2026-02-22T21:42:24.788 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5600 IOs, 2587792384 bytes 2026-02-22T21:42:24.826 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5700 IOs, 2612689920 bytes 2026-02-22T21:42:24.868 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5800 IOs, 2648257536 bytes 2026-02-22T21:42:24.901 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5900 IOs, 2665106432 bytes 2026-02-22T21:42:24.931 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6000 IOs, 2669086720 bytes 2026-02-22T21:42:24.958 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6100 IOs, 2669681664 bytes 2026-02-22T21:42:25.014 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6200 IOs, 2743544832 bytes 2026-02-22T21:42:25.053 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6300 IOs, 2759413760 bytes 2026-02-22T21:42:25.096 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6400 IOs, 2802247680 bytes 2026-02-22T21:42:25.155 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6500 IOs, 2870580224 bytes 2026-02-22T21:42:25.195 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6600 IOs, 2891472896 bytes 2026-02-22T21:42:25.233 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6700 IOs, 2924814336 bytes 2026-02-22T21:42:25.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6800 IOs, 2957983744 bytes 2026-02-22T21:42:25.328 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6900 IOs, 3022433280 bytes 2026-02-22T21:42:25.380 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7000 IOs, 3055202304 bytes 2026-02-22T21:42:25.419 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7100 IOs, 3091174400 bytes 2026-02-22T21:42:25.467 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7200 IOs, 3141938176 bytes 2026-02-22T21:42:25.513 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7300 IOs, 3173039104 bytes 2026-02-22T21:42:25.552 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7400 IOs, 3197144064 bytes 2026-02-22T21:42:25.601 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7500 IOs, 3250845696 bytes 2026-02-22T21:42:25.639 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7600 IOs, 3279926272 bytes 2026-02-22T21:42:25.677 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7700 IOs, 3305634816 bytes 2026-02-22T21:42:25.724 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7800 IOs, 3347141632 bytes 2026-02-22T21:42:25.773 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7900 IOs, 3402558464 bytes 2026-02-22T21:42:25.821 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8000 IOs, 3474498560 bytes 2026-02-22T21:42:25.889 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8100 IOs, 3537958912 bytes 2026-02-22T21:42:25.939 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8200 IOs, 3600105472 bytes 2026-02-22T21:42:25.981 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.5892094748181876) > weight (0.25) 2026-02-22T21:42:25.982 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.46625291674084135) > weight (0.25) 2026-02-22T21:42:25.982 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.6908523134568199) > weight (0.25) 2026-02-22T21:42:25.982 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.3 with skip (0.2922528164031176) > weight (0.25) 2026-02-22T21:42:25.982 INFO:tasks.rbd_mirror_thrash:waiting for 78.95291219727345 secs before thrashing 2026-02-22T21:42:25.983 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8300 IOs, 3624411136 bytes 2026-02-22T21:42:26.009 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8400 IOs, 3625306112 bytes 2026-02-22T21:42:26.047 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8500 IOs, 3654547456 bytes 2026-02-22T21:42:26.098 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8600 IOs, 3712290816 bytes 2026-02-22T21:42:26.160 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8700 IOs, 3796670464 bytes 2026-02-22T21:42:26.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8800 IOs, 3831292928 bytes 2026-02-22T21:42:26.241 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8900 IOs, 3862225920 bytes 2026-02-22T21:42:26.304 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9000 IOs, 3943004160 bytes 2026-02-22T21:42:26.353 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9100 IOs, 3987840000 bytes 2026-02-22T21:42:26.403 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9200 IOs, 4045539328 bytes 2026-02-22T21:42:26.445 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9300 IOs, 4076894208 bytes 2026-02-22T21:42:26.484 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9400 IOs, 4102598656 bytes 2026-02-22T21:42:26.525 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9500 IOs, 4138596352 bytes 2026-02-22T21:42:26.569 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9600 IOs, 4182695936 bytes 2026-02-22T21:42:26.633 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9700 IOs, 4263548928 bytes 2026-02-22T21:42:26.697 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9800 IOs, 4348820480 bytes 2026-02-22T21:42:26.761 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9900 IOs, 4423003136 bytes 2026-02-22T21:42:26.809 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10000 IOs, 4471300096 bytes 2026-02-22T21:42:26.848 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10100 IOs, 4502766592 bytes 2026-02-22T21:42:26.904 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10200 IOs, 4583129088 bytes 2026-02-22T21:42:26.964 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10300 IOs, 4654190592 bytes 2026-02-22T21:42:27.020 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10400 IOs, 4701996032 bytes 2026-02-22T21:42:27.059 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10500 IOs, 4748362752 bytes 2026-02-22T21:42:27.128 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10600 IOs, 4820538368 bytes 2026-02-22T21:42:27.166 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10700 IOs, 4846815232 bytes 2026-02-22T21:42:27.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10800 IOs, 4867375104 bytes 2026-02-22T21:42:27.239 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10900 IOs, 4894551040 bytes 2026-02-22T21:42:27.294 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11000 IOs, 4949553152 bytes 2026-02-22T21:42:27.355 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11100 IOs, 5011125248 bytes 2026-02-22T21:42:27.395 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11200 IOs, 5036803072 bytes 2026-02-22T21:42:27.431 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11300 IOs, 5068098560 bytes 2026-02-22T21:42:27.462 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11400 IOs, 5072232448 bytes 2026-02-22T21:42:27.507 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11500 IOs, 5119028224 bytes 2026-02-22T21:42:27.549 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11600 IOs, 5151727616 bytes 2026-02-22T21:42:27.600 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11700 IOs, 5209389056 bytes 2026-02-22T21:42:27.653 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11800 IOs, 5267138560 bytes 2026-02-22T21:42:27.697 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11900 IOs, 5308413952 bytes 2026-02-22T21:42:27.741 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12000 IOs, 5357290496 bytes 2026-02-22T21:42:27.792 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12100 IOs, 5413249024 bytes 2026-02-22T21:42:27.829 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12200 IOs, 5413877760 bytes 2026-02-22T21:42:27.864 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12300 IOs, 5429911552 bytes 2026-02-22T21:42:27.900 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12400 IOs, 5451707392 bytes 2026-02-22T21:42:27.942 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12500 IOs, 5486337024 bytes 2026-02-22T21:42:27.976 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12600 IOs, 5499586560 bytes 2026-02-22T21:42:28.013 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12700 IOs, 5517151232 bytes 2026-02-22T21:42:28.041 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12800 IOs, 5517751296 bytes 2026-02-22T21:42:28.081 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12900 IOs, 5547761664 bytes 2026-02-22T21:42:28.162 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13000 IOs, 5669606400 bytes 2026-02-22T21:42:28.214 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13100 IOs, 5723056128 bytes 2026-02-22T21:42:28.264 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13200 IOs, 5774917632 bytes 2026-02-22T21:42:28.298 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13300 IOs, 5788024832 bytes 2026-02-22T21:42:28.350 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13400 IOs, 5842199552 bytes 2026-02-22T21:42:28.402 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13500 IOs, 5899725824 bytes 2026-02-22T21:42:28.440 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13600 IOs, 5929972736 bytes 2026-02-22T21:42:28.477 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13700 IOs, 5951290368 bytes 2026-02-22T21:42:28.543 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13800 IOs, 6039102464 bytes 2026-02-22T21:42:28.590 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13900 IOs, 6093910016 bytes 2026-02-22T21:42:28.627 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14000 IOs, 6101356544 bytes 2026-02-22T21:42:28.675 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14100 IOs, 6160303104 bytes 2026-02-22T21:42:28.740 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14200 IOs, 6243758080 bytes 2026-02-22T21:42:28.793 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14300 IOs, 6293403648 bytes 2026-02-22T21:42:28.848 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14400 IOs, 6359655424 bytes 2026-02-22T21:42:28.904 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14500 IOs, 6425981952 bytes 2026-02-22T21:42:28.969 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14600 IOs, 6513224704 bytes 2026-02-22T21:42:29.003 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14700 IOs, 6522715136 bytes 2026-02-22T21:42:29.060 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14800 IOs, 6582445056 bytes 2026-02-22T21:42:29.106 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14900 IOs, 6635812864 bytes 2026-02-22T21:42:29.154 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15000 IOs, 6673501184 bytes 2026-02-22T21:42:29.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15100 IOs, 6740450304 bytes 2026-02-22T21:42:29.257 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15200 IOs, 6793716736 bytes 2026-02-22T21:42:29.306 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15300 IOs, 6825700352 bytes 2026-02-22T21:42:29.337 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15400 IOs, 6826302464 bytes 2026-02-22T21:42:29.372 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15500 IOs, 6845336576 bytes 2026-02-22T21:42:29.399 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15600 IOs, 6845962240 bytes 2026-02-22T21:42:29.439 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15700 IOs, 6885753856 bytes 2026-02-22T21:42:29.492 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15800 IOs, 6942263296 bytes 2026-02-22T21:42:29.539 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15900 IOs, 6988236800 bytes 2026-02-22T21:42:29.579 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16000 IOs, 7021041664 bytes 2026-02-22T21:42:29.609 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16100 IOs, 7022588928 bytes 2026-02-22T21:42:29.648 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16200 IOs, 7044803584 bytes 2026-02-22T21:42:29.701 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16300 IOs, 7104162816 bytes 2026-02-22T21:42:29.750 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16400 IOs, 7160958976 bytes 2026-02-22T21:42:29.796 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16500 IOs, 7204816896 bytes 2026-02-22T21:42:29.837 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16600 IOs, 7218381824 bytes 2026-02-22T21:42:29.879 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16700 IOs, 7269010432 bytes 2026-02-22T21:42:29.915 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16800 IOs, 7279611904 bytes 2026-02-22T21:42:29.983 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16900 IOs, 7388437504 bytes 2026-02-22T21:42:30.049 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17000 IOs, 7456483328 bytes 2026-02-22T21:42:30.079 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17100 IOs, 7458204672 bytes 2026-02-22T21:42:30.113 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17200 IOs, 7480228864 bytes 2026-02-22T21:42:30.156 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17300 IOs, 7509525504 bytes 2026-02-22T21:42:30.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17400 IOs, 7534133248 bytes 2026-02-22T21:42:30.248 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17500 IOs, 7601860608 bytes 2026-02-22T21:42:30.303 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17600 IOs, 7659267072 bytes 2026-02-22T21:42:30.352 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17700 IOs, 7723852800 bytes 2026-02-22T21:42:30.411 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17800 IOs, 7778839552 bytes 2026-02-22T21:42:30.484 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17900 IOs, 7895797760 bytes 2026-02-22T21:42:30.527 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18000 IOs, 7932850176 bytes 2026-02-22T21:42:30.590 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18100 IOs, 8008381440 bytes 2026-02-22T21:42:30.633 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18200 IOs, 8048373760 bytes 2026-02-22T21:42:30.671 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18300 IOs, 8074326016 bytes 2026-02-22T21:42:30.706 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18400 IOs, 8091184128 bytes 2026-02-22T21:42:30.738 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18500 IOs, 8106110976 bytes 2026-02-22T21:42:30.777 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18600 IOs, 8126202880 bytes 2026-02-22T21:42:30.812 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18700 IOs, 8150548480 bytes 2026-02-22T21:42:30.849 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18800 IOs, 8173655040 bytes 2026-02-22T21:42:30.878 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18900 IOs, 8174270464 bytes 2026-02-22T21:42:30.904 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19000 IOs, 8174865408 bytes 2026-02-22T21:42:30.941 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19100 IOs, 8196370432 bytes 2026-02-22T21:42:30.979 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19200 IOs, 8221360128 bytes 2026-02-22T21:42:31.024 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19300 IOs, 8253630464 bytes 2026-02-22T21:42:31.084 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19400 IOs, 8336275456 bytes 2026-02-22T21:42:31.130 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19500 IOs, 8377108480 bytes 2026-02-22T21:42:31.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19600 IOs, 8413162496 bytes 2026-02-22T21:42:31.238 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19700 IOs, 8487287808 bytes 2026-02-22T21:42:31.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19800 IOs, 8509576192 bytes 2026-02-22T21:42:31.320 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19900 IOs, 8549024768 bytes 2026-02-22T21:42:31.363 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20000 IOs, 8577320960 bytes 2026-02-22T21:42:31.417 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20100 IOs, 8641490944 bytes 2026-02-22T21:42:31.449 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20200 IOs, 8673038336 bytes 2026-02-22T21:42:31.495 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20300 IOs, 8691693568 bytes 2026-02-22T21:42:31.569 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20400 IOs, 8795569152 bytes 2026-02-22T21:42:31.601 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20500 IOs, 8808912896 bytes 2026-02-22T21:42:31.661 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20600 IOs, 8877901824 bytes 2026-02-22T21:42:31.704 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20700 IOs, 8898307072 bytes 2026-02-22T21:42:31.945 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20800 IOs, 8937463808 bytes 2026-02-22T21:42:32.196 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20900 IOs, 8997056512 bytes 2026-02-22T21:42:32.247 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21000 IOs, 9035846656 bytes 2026-02-22T21:42:32.274 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21100 IOs, 9037295616 bytes 2026-02-22T21:42:32.336 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21200 IOs, 9117139968 bytes 2026-02-22T21:42:32.383 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21300 IOs, 9149720576 bytes 2026-02-22T21:42:32.441 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21400 IOs, 9216635904 bytes 2026-02-22T21:42:32.491 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21500 IOs, 9266976768 bytes 2026-02-22T21:42:32.536 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21600 IOs, 9302819840 bytes 2026-02-22T21:42:32.578 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21700 IOs, 9328391168 bytes 2026-02-22T21:42:32.629 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21800 IOs, 9388476416 bytes 2026-02-22T21:42:32.678 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21900 IOs, 9432843264 bytes 2026-02-22T21:42:32.731 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22000 IOs, 9487703040 bytes 2026-02-22T21:42:32.771 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22100 IOs, 9524984832 bytes 2026-02-22T21:42:32.829 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22200 IOs, 9589042176 bytes 2026-02-22T21:42:32.862 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22300 IOs, 9595864064 bytes 2026-02-22T21:42:32.894 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22400 IOs, 9615545344 bytes 2026-02-22T21:42:32.933 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22500 IOs, 9646291968 bytes 2026-02-22T21:42:32.984 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22600 IOs, 9683249152 bytes 2026-02-22T21:42:33.050 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22700 IOs, 9774162944 bytes 2026-02-22T21:42:33.089 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22800 IOs, 9796886528 bytes 2026-02-22T21:42:33.141 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22900 IOs, 9846082560 bytes 2026-02-22T21:42:33.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23000 IOs, 9895399424 bytes 2026-02-22T21:42:33.224 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23100 IOs, 9907016704 bytes 2026-02-22T21:42:33.264 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23200 IOs, 9936349184 bytes 2026-02-22T21:42:33.306 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23300 IOs, 9972704256 bytes 2026-02-22T21:42:33.364 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23400 IOs, 10075273216 bytes 2026-02-22T21:42:33.435 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23500 IOs, 10149998592 bytes 2026-02-22T21:42:33.480 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23600 IOs, 10189445120 bytes 2026-02-22T21:42:33.520 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23700 IOs, 10219657216 bytes 2026-02-22T21:42:33.561 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23800 IOs, 10247474176 bytes 2026-02-22T21:42:33.611 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23900 IOs, 10300876800 bytes 2026-02-22T21:42:33.664 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24000 IOs, 10365941760 bytes 2026-02-22T21:42:33.695 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24100 IOs, 10384104448 bytes 2026-02-22T21:42:33.775 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24200 IOs, 10482967552 bytes 2026-02-22T21:42:33.813 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24300 IOs, 10502934528 bytes 2026-02-22T21:42:33.840 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24400 IOs, 10503550976 bytes 2026-02-22T21:42:33.877 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24500 IOs, 10538959872 bytes 2026-02-22T21:42:33.910 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24600 IOs, 10539587584 bytes 2026-02-22T21:42:33.975 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24700 IOs, 10630076416 bytes 2026-02-22T21:42:34.036 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24800 IOs, 10720727040 bytes 2026-02-22T21:42:34.083 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24900 IOs, 10743518208 bytes 2026-02-22T21:42:34.140 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25000 IOs, 10801372160 bytes 2026-02-22T21:42:34.180 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25100 IOs, 10836948992 bytes 2026-02-22T21:42:34.242 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25200 IOs, 10907582464 bytes 2026-02-22T21:42:34.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25300 IOs, 10915291136 bytes 2026-02-22T21:42:34.319 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25400 IOs, 10956857344 bytes 2026-02-22T21:42:34.368 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25500 IOs, 11000711168 bytes 2026-02-22T21:42:34.397 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25600 IOs, 11001328640 bytes 2026-02-22T21:42:34.432 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25700 IOs, 11019869184 bytes 2026-02-22T21:42:34.480 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25800 IOs, 11072296960 bytes 2026-02-22T21:42:34.514 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25900 IOs, 11086172160 bytes 2026-02-22T21:42:34.560 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:26000 IOs, 11124036608 bytes 2026-02-22T21:42:34.614 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:26100 IOs, 11175184384 bytes 2026-02-22T21:42:34.651 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:26200 IOs, 11212167168 bytes 2026-02-22T21:42:34.700 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:26300 IOs, 11258061824 bytes 2026-02-22T21:42:34.752 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:26400 IOs, 11313304576 bytes 2026-02-22T21:42:34.820 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:26500 IOs, 11406351360 bytes 2026-02-22T21:42:34.852 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:26600 IOs, 11416680448 bytes 2026-02-22T21:42:34.897 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:26700 IOs, 11457268736 bytes 2026-02-22T21:42:34.937 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:26800 IOs, 11490960384 bytes 2026-02-22T21:42:34.978 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:26900 IOs, 11522604032 bytes 2026-02-22T21:42:35.031 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:27000 IOs, 11580193792 bytes 2026-02-22T21:42:35.054 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ error_code=124 2026-02-22T21:42:35.054 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set -e 2026-02-22T21:42:35.054 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' 124 -eq 124 ']' 2026-02-22T21:42:35.054 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:42:35.054 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T21:42:35.055 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:42:35.055 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:42:35.055 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:42:35.055 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state_pattern=up+replaying 2026-02-22T21:42:35.055 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local description_pattern= 2026-02-22T21:42:35.055 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local service_pattern= 2026-02-22T21:42:35.055 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:42:35.055 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T21:42:36.055 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T21:42:36.055 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:42:36.055 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:42:36.055 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:42:36.055 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state_pattern=up+replaying 2026-02-22T21:42:36.055 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local description_pattern= 2026-02-22T21:42:36.055 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local service_pattern= 2026-02-22T21:42:36.055 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status 2026-02-22T21:42:36.056 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ CEPH_ARGS= 2026-02-22T21:42:36.056 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:42:36.089 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ status='test: 2026-02-22T21:42:36.089 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: global_id: b410963b-cae9-4337-a6ef-da65321c9374 2026-02-22T21:42:36.089 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: state: up+replaying 2026-02-22T21:42:36.089 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: description: replaying, {"bytes_per_second":17875694.93,"bytes_per_snapshot":536345600.0,"last_snapshot_bytes":536270848,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771796534,"remote_snapshot_timestamp":1771796534,"replay_state":"idle"} 2026-02-22T21:42:36.090 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: service: mirror.0 on trial026 2026-02-22T21:42:36.090 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: last_update: 2026-02-22 21:42:32 2026-02-22T21:42:36.090 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: peer_sites: 2026-02-22T21:42:36.090 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: name: cluster2 2026-02-22T21:42:36.090 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: state: up+stopped 2026-02-22T21:42:36.090 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: description: local image is primary 2026-02-22T21:42:36.090 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: last_update: 2026-02-22 21:42:33' 2026-02-22T21:42:36.090 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T21:42:36.090 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: state: up+replaying 2026-02-22T21:42:36.090 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep '^ description: .*' 2026-02-22T21:42:36.091 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: description: replaying, {"bytes_per_second":17875694.93,"bytes_per_snapshot":536345600.0,"last_snapshot_bytes":536270848,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771796534,"remote_snapshot_timestamp":1771796534,"replay_state":"idle"} 2026-02-22T21:42:36.091 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n '' ']' 2026-02-22T21:42:36.091 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ echo up+replaying 2026-02-22T21:42:36.091 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep '^up+' 2026-02-22T21:42:36.092 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:up+replaying 2026-02-22T21:42:36.092 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:42:36.093 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: service: mirror.0 on trial026 2026-02-22T21:42:36.093 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local last_update 2026-02-22T21:42:36.094 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:42:36.095 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ last_update='2026-02-22 21:42:32' 2026-02-22T21:42:36.095 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 21:42:32' 2026-02-22T21:42:36.095 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:42:36.096 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:42:36.096 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:42:36.096 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state_pattern=up+replaying 2026-02-22T21:42:36.096 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local 'prev_last_update=2026-02-22 21:42:32' 2026-02-22T21:42:36.096 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status 2026-02-22T21:42:36.096 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ CEPH_ARGS= 2026-02-22T21:42:36.096 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:42:36.127 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ status='OKOKOK142074258mirror.0trial02619.2.3-689-gf90f4eaetrueOK42104261mirror.1trial02619.2.3-689-gf90f4eaefalseOK42184265mirror.3trial02619.2.3-689-gf90f4eaefalseOK43584361mirror.2trial02619.2.3-689-gf90f4eaefalseOKtestb410963b-cae9-4337-a6ef-da65321c9374up+replayingreplaying, {"bytes_per_second":17875694.93,"bytes_per_snapshot":536345600.0,"last_snapshot_bytes":536270848,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771796534,"remote_snapshot_timestamp":1771796534,"replay_state":"idle"}42074258mirror.0trial0262026-02-22 21:42:32cluster2cfa81703-817a-4074-adbc-5e6e200790bfup+stoppedlocal image is primary2026-02-22 21:42:33' 2026-02-22T21:42:36.127 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local last_update state 2026-02-22T21:42:36.127 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:42:36.128 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ last_update='2026-02-22 21:42:32' 2026-02-22T21:42:36.129 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:42:36.130 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ state=up+replaying 2026-02-22T21:42:36.130 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ echo up+replaying 2026-02-22T21:42:36.130 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep up+replaying 2026-02-22T21:42:36.131 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:up+replaying 2026-02-22T21:42:36.132 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:42:36.132 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:42:36.132 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ snap_name=snap6 2026-02-22T21:42:36.132 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_snap cluster2 mirror test snap6 2026-02-22T21:42:36.132 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:42:36.132 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:42:36.132 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:42:36.132 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap6 2026-02-22T21:42:36.132 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap6 --debug-rbd=20 --debug-journaler=20 2026-02-22T21:42:40.159 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.9155475373703909) > weight (0.25) 2026-02-22T21:42:40.159 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.9277131803007571) > weight (0.25) 2026-02-22T21:42:40.160 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.367149838871929) > weight (0.25) 2026-02-22T21:42:40.160 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.3 2026-02-22T21:42:40.160 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Sent signal 15 2026-02-22T21:42:40.160 INFO:tasks.rbd_mirror_thrash:waiting for 6.512678495187352 secs before reviving daemons 2026-02-22T21:42:40.161 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial026.stderr:2026-02-22T21:42:40.160+0000 7f1e8f857640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.3 (PID: 48058) UID: 1000 2026-02-22T21:42:46.674 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.3 2026-02-22T21:42:46.674 INFO:teuthology.orchestra.run:waiting for 600 2026-02-22T21:42:46.675 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.3 2026-02-22T21:42:46.675 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Restarting daemon 2026-02-22T21:42:46.675 DEBUG:teuthology.orchestra.run.trial026:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.3 2026-02-22T21:42:46.678 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Started 2026-02-22T21:42:46.678 INFO:tasks.rbd_mirror_thrash:waiting for 101.65860097631281 secs before thrashing 2026-02-22T21:43:06.117 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T21:43:06.117 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:43:06.117 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:43:06.117 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:43:06.117 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T21:43:06.117 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:43:06.117 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:43:06.117 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:43:06.117 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=started 2026-02-22T21:43:06.117 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T21:43:06.117 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:43:06.117 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T21:43:07.119 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T21:43:07.119 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:43:07.119 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:43:07.120 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:43:07.120 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=started 2026-02-22T21:43:07.120 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_result 2026-02-22T21:43:07.120 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=stopped 2026-02-22T21:43:07.120 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:43:07.120 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_instance=cluster1 2026-02-22T21:43:07.120 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:43:07.120 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ grep -i state 2026-02-22T21:43:07.120 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T21:43:07.120 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance=cluster1 2026-02-22T21:43:07.120 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local loop_instance 2026-02-22T21:43:07.121 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:43:07.121 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sleep 0 2026-02-22T21:43:07.123 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:43:07.123 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ seq 0 3 2026-02-22T21:43:07.124 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:43:07.124 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:43:07.124 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:0 2026-02-22T21:43:07.124 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:43:07.124 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:43:07.124 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:43:07.124 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:0 2026-02-22T21:43:07.124 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:43:07.124 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:43:07.124 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:43:07.124 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=0 2026-02-22T21:43:07.125 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:43:07.125 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:43:07.125 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:43:07.125 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=0 2026-02-22T21:43:07.125 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T21:43:07.125 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:43:07.125 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:43:07.125 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:43:07.125 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:43:07.125 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:43:07.125 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:0 2026-02-22T21:43:07.125 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:43:07.125 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:43:07.126 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:43:07.126 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T21:43:07.126 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:43:07.126 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:43:07.126 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:43:07.126 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=0 2026-02-22T21:43:07.126 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=0 2026-02-22T21:43:07.126 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:43:07.134 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:43:07.134 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:43:07.134 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:43:07.134 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:43:07.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ return 0 2026-02-22T21:43:07.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:43:07.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:43:07.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep -i Replaying 2026-02-22T21:43:07.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: "state": "Replaying" 2026-02-22T21:43:07.184 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_state=started 2026-02-22T21:43:07.184 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test started = started 2026-02-22T21:43:07.184 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:43:07.184 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2026-02-22T21:43:07.184 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T21:43:07.184 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:43:07.184 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:43:07.184 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:43:07.184 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = journal ']' 2026-02-22T21:43:07.184 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T21:43:07.185 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-22T21:43:07.185 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:43:07.185 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:43:07.185 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:43:07.185 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-22T21:43:09.119 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Snapshot ID: 20 2026-02-22T21:43:09.128 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test 2026-02-22T21:43:09.128 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T21:43:09.128 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:43:09.129 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:43:09.129 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:43:09.129 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster2-mirror-test.status 2026-02-22T21:43:09.129 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster2-mirror-test.status 2026-02-22T21:43:09.129 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T21:43:09.130 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_log=/tmp/tmp.j44LZPowal/cluster2-mirror-test.status 2026-02-22T21:43:09.130 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster1-mirror-test.status 2026-02-22T21:43:09.130 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster1-mirror-test.status 2026-02-22T21:43:09.131 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T21:43:09.132 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_status_log=/tmp/tmp.j44LZPowal/cluster1-mirror-test.status 2026-02-22T21:43:09.132 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.j44LZPowal/cluster2-mirror-test.status 2026-02-22T21:43:09.132 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:43:09.132 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:43:09.133 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:43:09.133 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster2-mirror-test.status 2026-02-22T21:43:09.133 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-22T21:43:09.133 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:43:09.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T21:43:09.166 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snapshot_id=20 2026-02-22T21:43:09.166 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:43:09.166 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:43:09.166 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.2 2026-02-22T21:43:09.366 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.j44LZPowal/cluster1-mirror-test.status 2026-02-22T21:43:09.366 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:43:09.366 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:43:09.367 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:43:09.367 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-test.status 2026-02-22T21:43:09.367 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T21:43:09.367 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:43:09.397 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:43:09.397 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:43:09.398 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T21:43:09.398 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T21:43:09.398 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T21:43:09.399 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T21:43:09.399 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 20 = '' 2026-02-22T21:43:09.399 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:43:09.399 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.4 2026-02-22T21:43:09.799 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.j44LZPowal/cluster1-mirror-test.status 2026-02-22T21:43:09.799 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:43:09.799 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:43:09.799 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:43:09.800 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-test.status 2026-02-22T21:43:09.800 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T21:43:09.800 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:43:09.830 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:43:09.831 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:43:09.832 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T21:43:09.832 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T21:43:09.832 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T21:43:09.833 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T21:43:09.833 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 20 = '' 2026-02-22T21:43:09.833 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:43:09.833 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.8 2026-02-22T21:43:10.633 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.j44LZPowal/cluster1-mirror-test.status 2026-02-22T21:43:10.633 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:43:10.634 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:43:10.634 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:43:10.634 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-test.status 2026-02-22T21:43:10.634 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T21:43:10.634 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:43:10.669 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:43:10.670 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id=20 2026-02-22T21:43:10.670 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 20 = 20 2026-02-22T21:43:10.670 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:43:10.670 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snap_present cluster1 mirror test snap6 2026-02-22T21:43:10.670 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:43:10.670 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:43:10.670 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:43:10.670 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap6 2026-02-22T21:43:10.671 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T21:43:10.671 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T21:43:10.671 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T21:43:11.672 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 info mirror/test@snap6 2026-02-22T21:43:11.701 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'test': 2026-02-22T21:43:11.701 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 512 MiB in 128 objects 2026-02-22T21:43:11.701 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T21:43:11.701 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 12 2026-02-22T21:43:11.702 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 10a25aa14179 2026-02-22T21:43:11.702 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.10a25aa14179 2026-02-22T21:43:11.702 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T21:43:11.702 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock, non-primary 2026-02-22T21:43:11.702 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T21:43:11.702 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T21:43:11.702 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:38:05 2026 2026-02-22T21:43:11.702 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:41:25 2026 2026-02-22T21:43:11.702 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:38:05 2026 2026-02-22T21:43:11.702 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: protected: False 2026-02-22T21:43:11.703 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T21:43:11.703 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T21:43:11.703 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: b410963b-cae9-4337-a6ef-da65321c9374 2026-02-22T21:43:11.703 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: false 2026-02-22T21:43:11.704 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:43:11.704 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n snap5-clean ']' 2026-02-22T21:43:11.704 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ compare_image_snaps mirror test snap5-clean 2026-02-22T21:43:11.705 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:43:11.705 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:43:11.705 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap5-clean 2026-02-22T21:43:11.705 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local ret=0 2026-02-22T21:43:11.705 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local rmt_export=/tmp/tmp.j44LZPowal/cluster2-mirror-test.export 2026-02-22T21:43:11.705 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local loc_export=/tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:43:11.705 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:43:11.705 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror export test@snap5-clean /tmp/tmp.j44LZPowal/cluster2-mirror-test.export 2026-02-22T21:43:12.046 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-22T21:43:12.057 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 -p mirror export test@snap5-clean /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:43:12.480 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-22T21:43:12.492 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ cmp /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:43:12.671 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:43:12.793 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:43:12.793 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ compare_image_snaps mirror test snap6 2026-02-22T21:43:12.794 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:43:12.794 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:43:12.794 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap6 2026-02-22T21:43:12.794 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local ret=0 2026-02-22T21:43:12.794 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local rmt_export=/tmp/tmp.j44LZPowal/cluster2-mirror-test.export 2026-02-22T21:43:12.794 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local loc_export=/tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:43:12.794 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:43:12.795 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror export test@snap6 /tmp/tmp.j44LZPowal/cluster2-mirror-test.export 2026-02-22T21:43:13.121 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-22T21:43:13.132 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 -p mirror export test@snap6 /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:43:13.504 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-22T21:43:13.513 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ cmp /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:43:13.697 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:43:13.825 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:43:13.826 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ clean_snap_name=snap6-clean 2026-02-22T21:43:13.826 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_snap cluster2 mirror test snap6-clean 2026-02-22T21:43:13.826 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:43:13.826 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:43:13.826 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:43:13.826 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap6-clean 2026-02-22T21:43:13.826 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap6-clean --debug-rbd=20 --debug-journaler=20 2026-02-22T21:43:14.034 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 10` 2026-02-22T21:43:14.034 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ stress_write_image cluster2 mirror test 2026-02-22T21:43:14.034 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:43:14.034 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:43:14.034 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:43:14.034 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-22T21:43:14.035 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local duration=10 2026-02-22T21:43:14.035 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set +e 2026-02-22T21:43:14.036 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ timeout 10s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2026-02-22T21:43:14.146 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:100 IOs, 46049280 bytes 2026-02-22T21:43:14.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:200 IOs, 62204928 bytes 2026-02-22T21:43:14.238 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:300 IOs, 76014592 bytes 2026-02-22T21:43:14.281 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:400 IOs, 102812672 bytes 2026-02-22T21:43:14.312 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:500 IOs, 104269824 bytes 2026-02-22T21:43:14.362 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:600 IOs, 151098368 bytes 2026-02-22T21:43:14.399 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:700 IOs, 172891136 bytes 2026-02-22T21:43:14.439 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:800 IOs, 204844032 bytes 2026-02-22T21:43:14.476 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:900 IOs, 230821888 bytes 2026-02-22T21:43:14.522 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1000 IOs, 269438976 bytes 2026-02-22T21:43:14.566 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1100 IOs, 304315392 bytes 2026-02-22T21:43:14.598 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1200 IOs, 323290112 bytes 2026-02-22T21:43:14.652 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1300 IOs, 414040064 bytes 2026-02-22T21:43:14.726 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1400 IOs, 472096768 bytes 2026-02-22T21:43:14.770 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1500 IOs, 498947072 bytes 2026-02-22T21:43:14.811 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1600 IOs, 539094016 bytes 2026-02-22T21:43:14.844 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1700 IOs, 561793024 bytes 2026-02-22T21:43:14.883 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1800 IOs, 575454208 bytes 2026-02-22T21:43:14.923 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1900 IOs, 587476992 bytes 2026-02-22T21:43:14.975 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2000 IOs, 643878912 bytes 2026-02-22T21:43:15.031 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2100 IOs, 735933440 bytes 2026-02-22T21:43:15.097 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2200 IOs, 797043712 bytes 2026-02-22T21:43:15.138 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2300 IOs, 820368384 bytes 2026-02-22T21:43:15.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2400 IOs, 875663360 bytes 2026-02-22T21:43:15.223 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2500 IOs, 894763008 bytes 2026-02-22T21:43:15.277 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2600 IOs, 956692480 bytes 2026-02-22T21:43:15.319 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2700 IOs, 983954432 bytes 2026-02-22T21:43:15.576 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2800 IOs, 1047093248 bytes 2026-02-22T21:43:15.621 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2900 IOs, 1084716032 bytes 2026-02-22T21:43:15.663 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3000 IOs, 1118285824 bytes 2026-02-22T21:43:15.722 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3100 IOs, 1191898112 bytes 2026-02-22T21:43:15.755 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3200 IOs, 1206882304 bytes 2026-02-22T21:43:15.796 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3300 IOs, 1236636672 bytes 2026-02-22T21:43:15.831 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3400 IOs, 1250319360 bytes 2026-02-22T21:43:15.868 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3500 IOs, 1281644544 bytes 2026-02-22T21:43:15.915 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3600 IOs, 1319987200 bytes 2026-02-22T21:43:15.964 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3700 IOs, 1379428352 bytes 2026-02-22T21:43:16.011 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3800 IOs, 1404800000 bytes 2026-02-22T21:43:16.040 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3900 IOs, 1406709760 bytes 2026-02-22T21:43:16.102 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4000 IOs, 1495716864 bytes 2026-02-22T21:43:16.170 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4100 IOs, 1592339456 bytes 2026-02-22T21:43:16.219 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4200 IOs, 1623724032 bytes 2026-02-22T21:43:16.277 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4300 IOs, 1683079168 bytes 2026-02-22T21:43:16.318 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4400 IOs, 1711365120 bytes 2026-02-22T21:43:16.370 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4500 IOs, 1765237760 bytes 2026-02-22T21:43:16.422 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4600 IOs, 1827909632 bytes 2026-02-22T21:43:16.459 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4700 IOs, 1856094208 bytes 2026-02-22T21:43:16.505 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4800 IOs, 1895663616 bytes 2026-02-22T21:43:16.556 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4900 IOs, 1963475968 bytes 2026-02-22T21:43:16.611 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5000 IOs, 2006625280 bytes 2026-02-22T21:43:16.656 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5100 IOs, 2046155776 bytes 2026-02-22T21:43:16.688 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5200 IOs, 2068768768 bytes 2026-02-22T21:43:16.726 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5300 IOs, 2086353920 bytes 2026-02-22T21:43:16.773 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5400 IOs, 2133232640 bytes 2026-02-22T21:43:16.835 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5500 IOs, 2211222528 bytes 2026-02-22T21:43:16.886 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5600 IOs, 2259002368 bytes 2026-02-22T21:43:16.912 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5700 IOs, 2259603456 bytes 2026-02-22T21:43:16.947 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5800 IOs, 2280706048 bytes 2026-02-22T21:43:16.987 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5900 IOs, 2312823808 bytes 2026-02-22T21:43:17.042 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6000 IOs, 2376947712 bytes 2026-02-22T21:43:17.088 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6100 IOs, 2401263616 bytes 2026-02-22T21:43:17.120 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6200 IOs, 2404874240 bytes 2026-02-22T21:43:17.187 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6300 IOs, 2489735168 bytes 2026-02-22T21:43:17.244 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6400 IOs, 2589409280 bytes 2026-02-22T21:43:17.303 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6500 IOs, 2621455360 bytes 2026-02-22T21:43:17.345 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6600 IOs, 2652896256 bytes 2026-02-22T21:43:17.388 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6700 IOs, 2687983616 bytes 2026-02-22T21:43:17.438 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6800 IOs, 2740742144 bytes 2026-02-22T21:43:17.502 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6900 IOs, 2826507264 bytes 2026-02-22T21:43:17.554 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7000 IOs, 2894464000 bytes 2026-02-22T21:43:17.598 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7100 IOs, 2917455872 bytes 2026-02-22T21:43:17.654 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7200 IOs, 2982811648 bytes 2026-02-22T21:43:17.701 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7300 IOs, 3022455808 bytes 2026-02-22T21:43:17.748 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7400 IOs, 3066215424 bytes 2026-02-22T21:43:17.787 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7500 IOs, 3096294400 bytes 2026-02-22T21:43:17.839 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7600 IOs, 3160794112 bytes 2026-02-22T21:43:17.882 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7700 IOs, 3206002688 bytes 2026-02-22T21:43:17.933 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7800 IOs, 3236487168 bytes 2026-02-22T21:43:17.969 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7900 IOs, 3275738112 bytes 2026-02-22T21:43:18.035 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8000 IOs, 3364621312 bytes 2026-02-22T21:43:18.081 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8100 IOs, 3397476352 bytes 2026-02-22T21:43:18.124 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8200 IOs, 3426115584 bytes 2026-02-22T21:43:18.174 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8300 IOs, 3484087296 bytes 2026-02-22T21:43:18.222 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8400 IOs, 3534957568 bytes 2026-02-22T21:43:18.276 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8500 IOs, 3599263744 bytes 2026-02-22T21:43:18.329 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8600 IOs, 3661489152 bytes 2026-02-22T21:43:18.383 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8700 IOs, 3721775104 bytes 2026-02-22T21:43:18.432 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8800 IOs, 3766857728 bytes 2026-02-22T21:43:18.488 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8900 IOs, 3834676224 bytes 2026-02-22T21:43:18.532 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9000 IOs, 3857597440 bytes 2026-02-22T21:43:18.592 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9100 IOs, 3938594816 bytes 2026-02-22T21:43:18.642 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9200 IOs, 3969170432 bytes 2026-02-22T21:43:18.696 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9300 IOs, 4017781760 bytes 2026-02-22T21:43:18.729 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9400 IOs, 4041159680 bytes 2026-02-22T21:43:18.794 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9500 IOs, 4130599936 bytes 2026-02-22T21:43:18.855 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9600 IOs, 4199921664 bytes 2026-02-22T21:43:18.894 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9700 IOs, 4248298496 bytes 2026-02-22T21:43:18.927 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9800 IOs, 4250708992 bytes 2026-02-22T21:43:18.966 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9900 IOs, 4268341248 bytes 2026-02-22T21:43:19.010 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10000 IOs, 4299129856 bytes 2026-02-22T21:43:19.079 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10100 IOs, 4385427456 bytes 2026-02-22T21:43:19.131 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10200 IOs, 4434284544 bytes 2026-02-22T21:43:19.170 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10300 IOs, 4448529408 bytes 2026-02-22T21:43:19.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10400 IOs, 4495564800 bytes 2026-02-22T21:43:19.261 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10500 IOs, 4519284736 bytes 2026-02-22T21:43:19.293 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10600 IOs, 4519907328 bytes 2026-02-22T21:43:19.331 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10700 IOs, 4536038400 bytes 2026-02-22T21:43:19.375 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10800 IOs, 4563070976 bytes 2026-02-22T21:43:19.421 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10900 IOs, 4600233984 bytes 2026-02-22T21:43:19.457 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11000 IOs, 4614594560 bytes 2026-02-22T21:43:19.506 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11100 IOs, 4650523648 bytes 2026-02-22T21:43:19.564 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11200 IOs, 4713732096 bytes 2026-02-22T21:43:19.606 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11300 IOs, 4756004864 bytes 2026-02-22T21:43:19.679 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11400 IOs, 4848451584 bytes 2026-02-22T21:43:19.725 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11500 IOs, 4881029120 bytes 2026-02-22T21:43:19.766 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11600 IOs, 4911282176 bytes 2026-02-22T21:43:19.811 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11700 IOs, 4943390720 bytes 2026-02-22T21:43:19.873 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11800 IOs, 5012316160 bytes 2026-02-22T21:43:19.912 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11900 IOs, 5035058176 bytes 2026-02-22T21:43:19.953 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12000 IOs, 5077009408 bytes 2026-02-22T21:43:20.007 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12100 IOs, 5119065088 bytes 2026-02-22T21:43:20.039 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12200 IOs, 5119678464 bytes 2026-02-22T21:43:20.073 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12300 IOs, 5129228288 bytes 2026-02-22T21:43:20.118 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12400 IOs, 5151322112 bytes 2026-02-22T21:43:20.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12500 IOs, 5182240768 bytes 2026-02-22T21:43:20.213 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12600 IOs, 5220838400 bytes 2026-02-22T21:43:20.259 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12700 IOs, 5272428544 bytes 2026-02-22T21:43:20.307 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12800 IOs, 5305652224 bytes 2026-02-22T21:43:20.348 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12900 IOs, 5329797120 bytes 2026-02-22T21:43:20.416 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13000 IOs, 5433057280 bytes 2026-02-22T21:43:20.464 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13100 IOs, 5464780800 bytes 2026-02-22T21:43:20.505 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13200 IOs, 5478750208 bytes 2026-02-22T21:43:20.564 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13300 IOs, 5544015872 bytes 2026-02-22T21:43:20.612 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13400 IOs, 5583604736 bytes 2026-02-22T21:43:20.683 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13500 IOs, 5682701312 bytes 2026-02-22T21:43:20.720 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13600 IOs, 5701265408 bytes 2026-02-22T21:43:20.789 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13700 IOs, 5775935488 bytes 2026-02-22T21:43:20.829 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13800 IOs, 5804981248 bytes 2026-02-22T21:43:20.875 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13900 IOs, 5833664512 bytes 2026-02-22T21:43:20.908 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14000 IOs, 5834278912 bytes 2026-02-22T21:43:20.946 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14100 IOs, 5849160704 bytes 2026-02-22T21:43:20.989 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14200 IOs, 5883493376 bytes 2026-02-22T21:43:21.060 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14300 IOs, 5971996672 bytes 2026-02-22T21:43:21.117 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14400 IOs, 6032487424 bytes 2026-02-22T21:43:21.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14500 IOs, 6082889728 bytes 2026-02-22T21:43:21.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14600 IOs, 6106411008 bytes 2026-02-22T21:43:21.245 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14700 IOs, 6113628160 bytes 2026-02-22T21:43:21.312 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14800 IOs, 6196203520 bytes 2026-02-22T21:43:21.368 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14900 IOs, 6255327232 bytes 2026-02-22T21:43:21.406 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15000 IOs, 6272014336 bytes 2026-02-22T21:43:21.454 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15100 IOs, 6310975488 bytes 2026-02-22T21:43:21.504 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15200 IOs, 6354148352 bytes 2026-02-22T21:43:21.562 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15300 IOs, 6408343552 bytes 2026-02-22T21:43:21.624 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15400 IOs, 6486383616 bytes 2026-02-22T21:43:21.654 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15500 IOs, 6486981632 bytes 2026-02-22T21:43:21.697 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15600 IOs, 6515417088 bytes 2026-02-22T21:43:21.756 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15700 IOs, 6585922560 bytes 2026-02-22T21:43:21.823 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15800 IOs, 6654867456 bytes 2026-02-22T21:43:21.892 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15900 IOs, 6736312320 bytes 2026-02-22T21:43:21.944 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16000 IOs, 6784504832 bytes 2026-02-22T21:43:22.002 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16100 IOs, 6843247616 bytes 2026-02-22T21:43:22.045 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16200 IOs, 6870505472 bytes 2026-02-22T21:43:22.084 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16300 IOs, 6888321024 bytes 2026-02-22T21:43:22.117 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16400 IOs, 6892277760 bytes 2026-02-22T21:43:22.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16500 IOs, 6947317760 bytes 2026-02-22T21:43:22.235 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16600 IOs, 7024578560 bytes 2026-02-22T21:43:22.273 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16700 IOs, 7032416256 bytes 2026-02-22T21:43:22.323 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16800 IOs, 7075065856 bytes 2026-02-22T21:43:22.364 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16900 IOs, 7103502336 bytes 2026-02-22T21:43:22.423 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17000 IOs, 7146871808 bytes 2026-02-22T21:43:22.499 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17100 IOs, 7240565760 bytes 2026-02-22T21:43:22.548 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17200 IOs, 7266135040 bytes 2026-02-22T21:43:22.594 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17300 IOs, 7304729600 bytes 2026-02-22T21:43:22.636 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17400 IOs, 7335238656 bytes 2026-02-22T21:43:22.692 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17500 IOs, 7394109440 bytes 2026-02-22T21:43:22.725 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17600 IOs, 7394700288 bytes 2026-02-22T21:43:22.777 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17700 IOs, 7457648640 bytes 2026-02-22T21:43:22.821 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17800 IOs, 7468884992 bytes 2026-02-22T21:43:22.864 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17900 IOs, 7500750848 bytes 2026-02-22T21:43:22.911 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18000 IOs, 7535688704 bytes 2026-02-22T21:43:22.960 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18100 IOs, 7582298112 bytes 2026-02-22T21:43:23.023 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18200 IOs, 7665939456 bytes 2026-02-22T21:43:23.071 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18300 IOs, 7700216832 bytes 2026-02-22T21:43:23.112 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18400 IOs, 7713007616 bytes 2026-02-22T21:43:23.163 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18500 IOs, 7759341568 bytes 2026-02-22T21:43:23.215 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18600 IOs, 7790166016 bytes 2026-02-22T21:43:23.268 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18700 IOs, 7842869248 bytes 2026-02-22T21:43:23.310 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18800 IOs, 7865339904 bytes 2026-02-22T21:43:23.361 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18900 IOs, 7908180992 bytes 2026-02-22T21:43:23.406 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19000 IOs, 7957630976 bytes 2026-02-22T21:43:23.441 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19100 IOs, 7958249472 bytes 2026-02-22T21:43:23.491 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19200 IOs, 7999947776 bytes 2026-02-22T21:43:23.546 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19300 IOs, 8067565568 bytes 2026-02-22T21:43:23.593 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19400 IOs, 8090334208 bytes 2026-02-22T21:43:23.652 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19500 IOs, 8144785408 bytes 2026-02-22T21:43:23.687 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19600 IOs, 8158170112 bytes 2026-02-22T21:43:23.720 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19700 IOs, 8181992448 bytes 2026-02-22T21:43:23.789 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19800 IOs, 8247976960 bytes 2026-02-22T21:43:23.846 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19900 IOs, 8296766464 bytes 2026-02-22T21:43:23.890 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20000 IOs, 8328717312 bytes 2026-02-22T21:43:23.925 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20100 IOs, 8339039232 bytes 2026-02-22T21:43:23.967 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20200 IOs, 8364068864 bytes 2026-02-22T21:43:24.007 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20300 IOs, 8380410880 bytes 2026-02-22T21:43:24.040 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ error_code=124 2026-02-22T21:43:24.040 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set -e 2026-02-22T21:43:24.040 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' 124 -eq 124 ']' 2026-02-22T21:43:24.040 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:43:24.040 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T21:43:24.040 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:43:24.040 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:43:24.040 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:43:24.040 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state_pattern=up+replaying 2026-02-22T21:43:24.041 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local description_pattern= 2026-02-22T21:43:24.041 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local service_pattern= 2026-02-22T21:43:24.041 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:43:24.041 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T21:43:25.040 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T21:43:25.041 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:43:25.041 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:43:25.041 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:43:25.041 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state_pattern=up+replaying 2026-02-22T21:43:25.041 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local description_pattern= 2026-02-22T21:43:25.041 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local service_pattern= 2026-02-22T21:43:25.041 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status 2026-02-22T21:43:25.041 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ CEPH_ARGS= 2026-02-22T21:43:25.042 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:43:25.075 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ status='test: 2026-02-22T21:43:25.075 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: global_id: b410963b-cae9-4337-a6ef-da65321c9374 2026-02-22T21:43:25.075 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: state: up+replaying 2026-02-22T21:43:25.075 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":536345600.0,"last_snapshot_bytes":536270848,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771796534,"remote_snapshot_timestamp":1771796534,"replay_state":"idle"} 2026-02-22T21:43:25.075 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: service: mirror.0 on trial026 2026-02-22T21:43:25.075 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: last_update: 2026-02-22 21:43:02 2026-02-22T21:43:25.076 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: peer_sites: 2026-02-22T21:43:25.076 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: name: cluster2 2026-02-22T21:43:25.076 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: state: up+stopped 2026-02-22T21:43:25.076 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: description: local image is primary 2026-02-22T21:43:25.076 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: last_update: 2026-02-22 21:43:03' 2026-02-22T21:43:25.076 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T21:43:25.077 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: state: up+replaying 2026-02-22T21:43:25.077 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep '^ description: .*' 2026-02-22T21:43:25.077 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":536345600.0,"last_snapshot_bytes":536270848,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771796534,"remote_snapshot_timestamp":1771796534,"replay_state":"idle"} 2026-02-22T21:43:25.078 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n '' ']' 2026-02-22T21:43:25.078 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ echo up+replaying 2026-02-22T21:43:25.078 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep '^up+' 2026-02-22T21:43:25.078 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:up+replaying 2026-02-22T21:43:25.079 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:43:25.079 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: service: mirror.0 on trial026 2026-02-22T21:43:25.080 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local last_update 2026-02-22T21:43:25.080 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:43:25.081 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ last_update='2026-02-22 21:43:02' 2026-02-22T21:43:25.081 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 21:43:02' 2026-02-22T21:43:25.081 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:43:25.081 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:43:25.081 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:43:25.081 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state_pattern=up+replaying 2026-02-22T21:43:25.082 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local 'prev_last_update=2026-02-22 21:43:02' 2026-02-22T21:43:25.082 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status 2026-02-22T21:43:25.082 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ CEPH_ARGS= 2026-02-22T21:43:25.082 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:43:25.113 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ status='OKOKOK142074258mirror.0trial02619.2.3-689-gf90f4eaetrueOK42104261mirror.1trial02619.2.3-689-gf90f4eaefalseOK43584361mirror.2trial02619.2.3-689-gf90f4eaefalseOK44434446mirror.3trial02619.2.3-689-gf90f4eaefalseOKtestb410963b-cae9-4337-a6ef-da65321c9374up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":536345600.0,"last_snapshot_bytes":536270848,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771796534,"remote_snapshot_timestamp":1771796534,"replay_state":"idle"}42074258mirror.0trial0262026-02-22 21:43:02cluster2cfa81703-817a-4074-adbc-5e6e200790bfup+stoppedlocal image is primary2026-02-22 21:43:03' 2026-02-22T21:43:25.113 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local last_update state 2026-02-22T21:43:25.113 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:43:25.115 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ last_update='2026-02-22 21:43:02' 2026-02-22T21:43:25.115 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:43:25.116 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ state=up+replaying 2026-02-22T21:43:25.117 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ echo up+replaying 2026-02-22T21:43:25.117 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep up+replaying 2026-02-22T21:43:25.118 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:up+replaying 2026-02-22T21:43:25.118 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:43:25.118 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:43:25.118 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ snap_name=snap7 2026-02-22T21:43:25.118 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_snap cluster2 mirror test snap7 2026-02-22T21:43:25.118 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:43:25.118 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:43:25.118 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:43:25.118 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap7 2026-02-22T21:43:25.118 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap7 --debug-rbd=20 --debug-journaler=20 2026-02-22T21:43:44.936 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.0 2026-02-22T21:43:44.937 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Sent signal 15 2026-02-22T21:43:44.937 INFO:tasks.rbd_mirror_thrash:waiting for 7.2716166681401715 secs before reviving daemons 2026-02-22T21:43:44.938 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial026.stderr:2026-02-22T21:43:44.936+0000 7f44348d5640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 (PID: 51547) UID: 1000 2026-02-22T21:43:52.209 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.0 2026-02-22T21:43:52.209 INFO:teuthology.orchestra.run:waiting for 600 2026-02-22T21:43:52.210 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.0 2026-02-22T21:43:52.210 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Restarting daemon 2026-02-22T21:43:52.210 DEBUG:teuthology.orchestra.run.trial026:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 2026-02-22T21:43:52.212 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Started 2026-02-22T21:43:52.213 INFO:tasks.rbd_mirror_thrash:waiting for 73.72475116205749 secs before thrashing 2026-02-22T21:43:54.115 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T21:43:54.116 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:43:54.116 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:43:54.116 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:43:54.116 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T21:43:54.116 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:43:54.116 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:43:54.116 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:43:54.116 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=started 2026-02-22T21:43:54.116 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T21:43:54.116 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:43:54.116 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T21:43:55.118 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T21:43:55.118 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:43:55.118 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:43:55.118 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:43:55.118 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=started 2026-02-22T21:43:55.119 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_result 2026-02-22T21:43:55.119 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=stopped 2026-02-22T21:43:55.119 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:43:55.119 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_instance=cluster1 2026-02-22T21:43:55.119 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:43:55.119 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T21:43:55.119 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance=cluster1 2026-02-22T21:43:55.119 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local loop_instance 2026-02-22T21:43:55.119 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:43:55.119 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sleep 0 2026-02-22T21:43:55.119 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ grep -i state 2026-02-22T21:43:55.120 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:43:55.122 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ seq 0 3 2026-02-22T21:43:55.122 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:43:55.122 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:43:55.123 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:0 2026-02-22T21:43:55.123 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:43:55.123 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:43:55.123 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:43:55.123 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:0 2026-02-22T21:43:55.123 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:43:55.123 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:43:55.123 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:43:55.123 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=0 2026-02-22T21:43:55.123 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:43:55.123 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:43:55.123 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:43:55.124 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=0 2026-02-22T21:43:55.124 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T21:43:55.124 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:43:55.124 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:43:55.124 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:43:55.124 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:43:55.124 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:43:55.124 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:0 2026-02-22T21:43:55.124 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:43:55.124 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:43:55.124 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:43:55.124 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T21:43:55.125 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:43:55.125 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:43:55.125 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:43:55.125 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=0 2026-02-22T21:43:55.125 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=0 2026-02-22T21:43:55.125 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:43:55.133 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:43:55.133 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:43:55.133 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:43:55.133 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:43:55.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ return 0 2026-02-22T21:43:55.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:43:55.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:43:55.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep -i Replaying 2026-02-22T21:43:55.180 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: "state": "Replaying" 2026-02-22T21:43:55.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_state=started 2026-02-22T21:43:55.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test started = started 2026-02-22T21:43:55.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:43:55.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2026-02-22T21:43:55.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T21:43:55.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:43:55.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:43:55.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:43:55.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = journal ']' 2026-02-22T21:43:55.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T21:43:55.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-22T21:43:55.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:43:55.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:43:55.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:43:55.183 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-22T21:43:57.118 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Snapshot ID: 23 2026-02-22T21:43:57.127 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test 2026-02-22T21:43:57.128 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T21:43:57.128 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:43:57.128 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:43:57.128 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:43:57.128 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster2-mirror-test.status 2026-02-22T21:43:57.129 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster2-mirror-test.status 2026-02-22T21:43:57.129 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T21:43:57.129 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_log=/tmp/tmp.j44LZPowal/cluster2-mirror-test.status 2026-02-22T21:43:57.130 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster1-mirror-test.status 2026-02-22T21:43:57.130 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster1-mirror-test.status 2026-02-22T21:43:57.130 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T21:43:57.132 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_status_log=/tmp/tmp.j44LZPowal/cluster1-mirror-test.status 2026-02-22T21:43:57.132 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.j44LZPowal/cluster2-mirror-test.status 2026-02-22T21:43:57.132 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:43:57.132 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:43:57.132 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:43:57.132 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster2-mirror-test.status 2026-02-22T21:43:57.132 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-22T21:43:57.132 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:43:57.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T21:43:57.166 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snapshot_id=23 2026-02-22T21:43:57.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:43:57.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:43:57.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.2 2026-02-22T21:43:57.368 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.j44LZPowal/cluster1-mirror-test.status 2026-02-22T21:43:57.368 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:43:57.368 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:43:57.368 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:43:57.368 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-test.status 2026-02-22T21:43:57.369 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T21:43:57.369 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:43:57.402 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:43:57.402 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:43:57.403 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T21:43:57.403 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T21:43:57.403 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T21:43:57.404 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T21:43:57.404 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 23 = '' 2026-02-22T21:43:57.404 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:43:57.404 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.4 2026-02-22T21:43:57.805 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.j44LZPowal/cluster1-mirror-test.status 2026-02-22T21:43:57.805 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:43:57.805 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:43:57.805 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:43:57.806 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-test.status 2026-02-22T21:43:57.806 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T21:43:57.806 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:43:57.836 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:43:57.837 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:43:57.838 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T21:43:57.838 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T21:43:57.838 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T21:43:57.839 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T21:43:57.839 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 23 = '' 2026-02-22T21:43:57.839 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:43:57.839 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.8 2026-02-22T21:43:58.640 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.j44LZPowal/cluster1-mirror-test.status 2026-02-22T21:43:58.640 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:43:58.640 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:43:58.640 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:43:58.640 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-test.status 2026-02-22T21:43:58.641 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T21:43:58.641 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:43:58.686 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:43:58.686 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:43:58.687 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T21:43:58.687 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T21:43:58.687 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T21:43:58.688 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T21:43:58.688 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 23 = '' 2026-02-22T21:43:58.688 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:43:58.688 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1.6 2026-02-22T21:44:00.289 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.j44LZPowal/cluster1-mirror-test.status 2026-02-22T21:44:00.289 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:44:00.289 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:44:00.289 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:44:00.289 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-test.status 2026-02-22T21:44:00.289 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T21:44:00.290 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:44:00.320 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:44:00.322 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id=23 2026-02-22T21:44:00.322 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 23 = 23 2026-02-22T21:44:00.322 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:44:00.322 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snap_present cluster1 mirror test snap7 2026-02-22T21:44:00.322 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:44:00.322 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:44:00.322 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:44:00.323 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap7 2026-02-22T21:44:00.323 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T21:44:00.323 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T21:44:00.323 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T21:44:01.323 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 info mirror/test@snap7 2026-02-22T21:44:01.352 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'test': 2026-02-22T21:44:01.352 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 512 MiB in 128 objects 2026-02-22T21:44:01.352 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T21:44:01.353 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 14 2026-02-22T21:44:01.353 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 10a25aa14179 2026-02-22T21:44:01.353 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.10a25aa14179 2026-02-22T21:44:01.353 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T21:44:01.353 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock, non-primary 2026-02-22T21:44:01.353 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T21:44:01.353 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T21:44:01.353 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:38:05 2026 2026-02-22T21:44:01.353 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:43:12 2026 2026-02-22T21:44:01.353 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:38:05 2026 2026-02-22T21:44:01.354 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: protected: False 2026-02-22T21:44:01.354 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T21:44:01.354 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T21:44:01.354 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: b410963b-cae9-4337-a6ef-da65321c9374 2026-02-22T21:44:01.354 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: false 2026-02-22T21:44:01.355 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:44:01.355 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n snap6-clean ']' 2026-02-22T21:44:01.355 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ compare_image_snaps mirror test snap6-clean 2026-02-22T21:44:01.355 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:44:01.355 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:44:01.356 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap6-clean 2026-02-22T21:44:01.356 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local ret=0 2026-02-22T21:44:01.356 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local rmt_export=/tmp/tmp.j44LZPowal/cluster2-mirror-test.export 2026-02-22T21:44:01.356 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local loc_export=/tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:44:01.356 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:44:01.356 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror export test@snap6-clean /tmp/tmp.j44LZPowal/cluster2-mirror-test.export 2026-02-22T21:44:01.689 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-22T21:44:01.699 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 -p mirror export test@snap6-clean /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:44:02.092 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-22T21:44:02.104 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ cmp /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:44:02.283 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:44:02.410 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:44:02.410 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ compare_image_snaps mirror test snap7 2026-02-22T21:44:02.410 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:44:02.410 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:44:02.410 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap7 2026-02-22T21:44:02.410 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local ret=0 2026-02-22T21:44:02.410 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local rmt_export=/tmp/tmp.j44LZPowal/cluster2-mirror-test.export 2026-02-22T21:44:02.411 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local loc_export=/tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:44:02.411 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:44:02.411 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror export test@snap7 /tmp/tmp.j44LZPowal/cluster2-mirror-test.export 2026-02-22T21:44:02.747 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-22T21:44:02.756 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 -p mirror export test@snap7 /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:44:03.161 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-22T21:44:03.173 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ cmp /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:44:03.362 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:44:03.487 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:44:03.487 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ clean_snap_name=snap7-clean 2026-02-22T21:44:03.488 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_snap cluster2 mirror test snap7-clean 2026-02-22T21:44:03.488 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:44:03.488 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:44:03.488 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:44:03.488 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap7-clean 2026-02-22T21:44:03.488 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap7-clean --debug-rbd=20 --debug-journaler=20 2026-02-22T21:44:04.102 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 10` 2026-02-22T21:44:04.102 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ stress_write_image cluster2 mirror test 2026-02-22T21:44:04.102 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:44:04.102 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:44:04.102 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:44:04.103 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-22T21:44:04.104 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local duration=7 2026-02-22T21:44:04.104 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set +e 2026-02-22T21:44:04.104 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ timeout 7s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2026-02-22T21:44:04.218 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:100 IOs, 62391296 bytes 2026-02-22T21:44:04.291 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:200 IOs, 137418752 bytes 2026-02-22T21:44:04.333 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:300 IOs, 145155072 bytes 2026-02-22T21:44:04.365 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:400 IOs, 150639616 bytes 2026-02-22T21:44:04.416 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:500 IOs, 196947968 bytes 2026-02-22T21:44:04.463 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:600 IOs, 237948928 bytes 2026-02-22T21:44:04.512 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:700 IOs, 278713344 bytes 2026-02-22T21:44:04.557 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:800 IOs, 317026304 bytes 2026-02-22T21:44:04.593 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:900 IOs, 337226752 bytes 2026-02-22T21:44:04.631 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1000 IOs, 361894912 bytes 2026-02-22T21:44:04.661 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1100 IOs, 362504192 bytes 2026-02-22T21:44:04.690 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1200 IOs, 363116544 bytes 2026-02-22T21:44:04.744 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1300 IOs, 417512448 bytes 2026-02-22T21:44:04.800 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1400 IOs, 485687296 bytes 2026-02-22T21:44:04.836 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1500 IOs, 503853056 bytes 2026-02-22T21:44:04.865 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1600 IOs, 504458240 bytes 2026-02-22T21:44:04.904 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1700 IOs, 532624384 bytes 2026-02-22T21:44:04.941 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1800 IOs, 552029184 bytes 2026-02-22T21:44:05.010 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1900 IOs, 647574528 bytes 2026-02-22T21:44:05.042 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2000 IOs, 658987008 bytes 2026-02-22T21:44:05.093 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2100 IOs, 713133056 bytes 2026-02-22T21:44:05.134 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2200 IOs, 738455552 bytes 2026-02-22T21:44:05.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2300 IOs, 833893376 bytes 2026-02-22T21:44:05.261 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2400 IOs, 905073664 bytes 2026-02-22T21:44:05.315 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2500 IOs, 965189632 bytes 2026-02-22T21:44:05.363 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2600 IOs, 1007410176 bytes 2026-02-22T21:44:05.404 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2700 IOs, 1045600256 bytes 2026-02-22T21:44:05.460 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2800 IOs, 1094765568 bytes 2026-02-22T21:44:05.496 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2900 IOs, 1115358208 bytes 2026-02-22T21:44:05.545 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3000 IOs, 1160314880 bytes 2026-02-22T21:44:05.606 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3100 IOs, 1227743232 bytes 2026-02-22T21:44:05.639 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3200 IOs, 1240608768 bytes 2026-02-22T21:44:05.683 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3300 IOs, 1286304768 bytes 2026-02-22T21:44:05.714 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3400 IOs, 1296863232 bytes 2026-02-22T21:44:05.752 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3500 IOs, 1317052416 bytes 2026-02-22T21:44:05.801 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3600 IOs, 1372919808 bytes 2026-02-22T21:44:05.843 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3700 IOs, 1393435648 bytes 2026-02-22T21:44:05.894 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3800 IOs, 1449253888 bytes 2026-02-22T21:44:05.930 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3900 IOs, 1468476416 bytes 2026-02-22T21:44:05.969 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4000 IOs, 1501903872 bytes 2026-02-22T21:44:06.015 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4100 IOs, 1534302208 bytes 2026-02-22T21:44:06.054 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4200 IOs, 1565841408 bytes 2026-02-22T21:44:06.137 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4300 IOs, 1676553216 bytes 2026-02-22T21:44:06.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4400 IOs, 1739823104 bytes 2026-02-22T21:44:06.244 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4500 IOs, 1771522048 bytes 2026-02-22T21:44:06.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4600 IOs, 1786883072 bytes 2026-02-22T21:44:06.325 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4700 IOs, 1830830080 bytes 2026-02-22T21:44:06.376 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4800 IOs, 1877734400 bytes 2026-02-22T21:44:06.420 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4900 IOs, 1916116992 bytes 2026-02-22T21:44:06.470 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5000 IOs, 1953107968 bytes 2026-02-22T21:44:06.511 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5100 IOs, 1990939648 bytes 2026-02-22T21:44:06.566 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5200 IOs, 2040062976 bytes 2026-02-22T21:44:06.614 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5300 IOs, 2085733376 bytes 2026-02-22T21:44:06.655 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5400 IOs, 2107988992 bytes 2026-02-22T21:44:06.716 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5500 IOs, 2181293056 bytes 2026-02-22T21:44:06.762 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5600 IOs, 2226579456 bytes 2026-02-22T21:44:06.824 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5700 IOs, 2295660544 bytes 2026-02-22T21:44:06.854 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5800 IOs, 2304677888 bytes 2026-02-22T21:44:06.902 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5900 IOs, 2344990720 bytes 2026-02-22T21:44:06.944 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6000 IOs, 2372214784 bytes 2026-02-22T21:44:07.010 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6100 IOs, 2467141632 bytes 2026-02-22T21:44:07.051 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6200 IOs, 2504588288 bytes 2026-02-22T21:44:07.099 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6300 IOs, 2541410304 bytes 2026-02-22T21:44:07.162 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6400 IOs, 2624735232 bytes 2026-02-22T21:44:07.228 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6500 IOs, 2696986624 bytes 2026-02-22T21:44:07.269 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6600 IOs, 2722782208 bytes 2026-02-22T21:44:07.338 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6700 IOs, 2811569152 bytes 2026-02-22T21:44:07.373 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6800 IOs, 2830044160 bytes 2026-02-22T21:44:07.409 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6900 IOs, 2842769408 bytes 2026-02-22T21:44:07.467 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7000 IOs, 2912990208 bytes 2026-02-22T21:44:07.506 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7100 IOs, 2940656640 bytes 2026-02-22T21:44:07.566 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7200 IOs, 3000815616 bytes 2026-02-22T21:44:07.618 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7300 IOs, 3061371904 bytes 2026-02-22T21:44:07.659 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7400 IOs, 3085051904 bytes 2026-02-22T21:44:07.718 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7500 IOs, 3152390144 bytes 2026-02-22T21:44:07.780 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7600 IOs, 3225308160 bytes 2026-02-22T21:44:07.832 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7700 IOs, 3282273280 bytes 2026-02-22T21:44:07.877 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7800 IOs, 3321184256 bytes 2026-02-22T21:44:07.936 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7900 IOs, 3379627008 bytes 2026-02-22T21:44:07.963 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8000 IOs, 3385677824 bytes 2026-02-22T21:44:08.021 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8100 IOs, 3451414528 bytes 2026-02-22T21:44:08.067 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8200 IOs, 3496161280 bytes 2026-02-22T21:44:08.105 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8300 IOs, 3525971968 bytes 2026-02-22T21:44:08.163 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8400 IOs, 3570757632 bytes 2026-02-22T21:44:08.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8500 IOs, 3633864704 bytes 2026-02-22T21:44:08.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8600 IOs, 3714526208 bytes 2026-02-22T21:44:08.326 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8700 IOs, 3750939648 bytes 2026-02-22T21:44:08.383 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8800 IOs, 3816438784 bytes 2026-02-22T21:44:08.431 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8900 IOs, 3855426560 bytes 2026-02-22T21:44:08.464 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9000 IOs, 3862983680 bytes 2026-02-22T21:44:08.502 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9100 IOs, 3884475392 bytes 2026-02-22T21:44:08.537 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9200 IOs, 3914591232 bytes 2026-02-22T21:44:08.591 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9300 IOs, 3971623936 bytes 2026-02-22T21:44:08.664 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9400 IOs, 4041713664 bytes 2026-02-22T21:44:08.715 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9500 IOs, 4084244480 bytes 2026-02-22T21:44:08.760 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9600 IOs, 4122288128 bytes 2026-02-22T21:44:08.804 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9700 IOs, 4159167488 bytes 2026-02-22T21:44:08.855 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9800 IOs, 4202103808 bytes 2026-02-22T21:44:08.906 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9900 IOs, 4247802880 bytes 2026-02-22T21:44:08.942 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10000 IOs, 4277407744 bytes 2026-02-22T21:44:08.977 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10100 IOs, 4291048448 bytes 2026-02-22T21:44:09.031 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10200 IOs, 4350248960 bytes 2026-02-22T21:44:09.079 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10300 IOs, 4410460160 bytes 2026-02-22T21:44:09.116 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10400 IOs, 4415221760 bytes 2026-02-22T21:44:09.152 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10500 IOs, 4432960512 bytes 2026-02-22T21:44:09.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10600 IOs, 4499525632 bytes 2026-02-22T21:44:09.250 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10700 IOs, 4525339648 bytes 2026-02-22T21:44:09.303 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10800 IOs, 4582883328 bytes 2026-02-22T21:44:09.355 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10900 IOs, 4642305024 bytes 2026-02-22T21:44:09.402 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11000 IOs, 4675226624 bytes 2026-02-22T21:44:09.446 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11100 IOs, 4707518464 bytes 2026-02-22T21:44:09.488 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11200 IOs, 4744769536 bytes 2026-02-22T21:44:09.549 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11300 IOs, 4818361344 bytes 2026-02-22T21:44:09.611 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11400 IOs, 4905945088 bytes 2026-02-22T21:44:09.647 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11500 IOs, 4919989248 bytes 2026-02-22T21:44:09.682 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11600 IOs, 4930670592 bytes 2026-02-22T21:44:09.734 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11700 IOs, 4987307008 bytes 2026-02-22T21:44:09.787 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11800 IOs, 5046375424 bytes 2026-02-22T21:44:09.831 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11900 IOs, 5087115264 bytes 2026-02-22T21:44:09.878 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12000 IOs, 5128823808 bytes 2026-02-22T21:44:09.917 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12100 IOs, 5166895104 bytes 2026-02-22T21:44:09.967 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12200 IOs, 5200415744 bytes 2026-02-22T21:44:10.004 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12300 IOs, 5201021952 bytes 2026-02-22T21:44:10.052 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12400 IOs, 5246546944 bytes 2026-02-22T21:44:10.100 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12500 IOs, 5292570624 bytes 2026-02-22T21:44:10.147 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12600 IOs, 5344096256 bytes 2026-02-22T21:44:10.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12700 IOs, 5406279680 bytes 2026-02-22T21:44:10.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12800 IOs, 5447707648 bytes 2026-02-22T21:44:10.289 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12900 IOs, 5465882624 bytes 2026-02-22T21:44:10.339 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13000 IOs, 5514889216 bytes 2026-02-22T21:44:10.385 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13100 IOs, 5560790016 bytes 2026-02-22T21:44:10.440 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13200 IOs, 5611091968 bytes 2026-02-22T21:44:10.469 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13300 IOs, 5611695104 bytes 2026-02-22T21:44:10.499 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13400 IOs, 5612302336 bytes 2026-02-22T21:44:10.742 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13500 IOs, 5643248640 bytes 2026-02-22T21:44:10.775 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13600 IOs, 5653204992 bytes 2026-02-22T21:44:10.818 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13700 IOs, 5685301248 bytes 2026-02-22T21:44:10.868 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13800 IOs, 5736430592 bytes 2026-02-22T21:44:10.918 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13900 IOs, 5810387968 bytes 2026-02-22T21:44:10.974 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14000 IOs, 5858362368 bytes 2026-02-22T21:44:11.017 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14100 IOs, 5900212224 bytes 2026-02-22T21:44:11.049 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14200 IOs, 5902692352 bytes 2026-02-22T21:44:11.108 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ error_code=124 2026-02-22T21:44:11.109 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set -e 2026-02-22T21:44:11.109 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' 124 -eq 124 ']' 2026-02-22T21:44:11.109 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:44:11.109 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T21:44:11.109 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:44:11.109 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:44:11.109 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:44:11.109 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state_pattern=up+replaying 2026-02-22T21:44:11.109 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local description_pattern= 2026-02-22T21:44:11.109 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local service_pattern= 2026-02-22T21:44:11.109 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:44:11.109 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T21:44:12.109 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T21:44:12.109 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:44:12.109 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:44:12.109 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:44:12.110 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state_pattern=up+replaying 2026-02-22T21:44:12.110 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local description_pattern= 2026-02-22T21:44:12.110 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local service_pattern= 2026-02-22T21:44:12.110 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status 2026-02-22T21:44:12.110 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ CEPH_ARGS= 2026-02-22T21:44:12.110 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:44:12.347 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ status='test: 2026-02-22T21:44:12.347 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: global_id: b410963b-cae9-4337-a6ef-da65321c9374 2026-02-22T21:44:12.347 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: state: up+replaying 2026-02-22T21:44:12.347 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: description: replaying, {"bytes_per_second":17884398.93,"bytes_per_snapshot":536564224.0,"last_snapshot_bytes":536531968,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771796636,"remote_snapshot_timestamp":1771796636,"replay_state":"idle"} 2026-02-22T21:44:12.348 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: service: mirror.0 on trial026 2026-02-22T21:44:12.348 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: last_update: 2026-02-22 21:44:02 2026-02-22T21:44:12.348 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: peer_sites: 2026-02-22T21:44:12.348 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: name: cluster2 2026-02-22T21:44:12.348 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: state: up+stopped 2026-02-22T21:44:12.348 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: description: local image is primary 2026-02-22T21:44:12.348 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: last_update: 2026-02-22 21:44:03' 2026-02-22T21:44:12.348 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T21:44:12.349 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: state: up+replaying 2026-02-22T21:44:12.349 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep '^ description: .*' 2026-02-22T21:44:12.350 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: description: replaying, {"bytes_per_second":17884398.93,"bytes_per_snapshot":536564224.0,"last_snapshot_bytes":536531968,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771796636,"remote_snapshot_timestamp":1771796636,"replay_state":"idle"} 2026-02-22T21:44:12.350 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n '' ']' 2026-02-22T21:44:12.350 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ echo up+replaying 2026-02-22T21:44:12.350 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep '^up+' 2026-02-22T21:44:12.351 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:up+replaying 2026-02-22T21:44:12.351 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:44:12.352 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: service: mirror.0 on trial026 2026-02-22T21:44:12.352 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local last_update 2026-02-22T21:44:12.352 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:44:12.354 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ last_update='2026-02-22 21:44:02' 2026-02-22T21:44:12.354 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 21:44:02' 2026-02-22T21:44:12.354 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:44:12.354 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:44:12.354 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:44:12.354 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state_pattern=up+replaying 2026-02-22T21:44:12.355 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local 'prev_last_update=2026-02-22 21:44:02' 2026-02-22T21:44:12.355 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status 2026-02-22T21:44:12.355 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ CEPH_ARGS= 2026-02-22T21:44:12.355 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:44:12.386 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ status='OKOKOK142074258mirror.0trial02619.2.3-689-gf90f4eaetrueOK42104261mirror.1trial02619.2.3-689-gf90f4eaefalseOK43584361mirror.2trial02619.2.3-689-gf90f4eaefalseOK44434446mirror.3trial02619.2.3-689-gf90f4eaefalseOKtestb410963b-cae9-4337-a6ef-da65321c9374up+replayingreplaying, {"bytes_per_second":17884398.93,"bytes_per_snapshot":536564224.0,"last_snapshot_bytes":536531968,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771796636,"remote_snapshot_timestamp":1771796636,"replay_state":"idle"}42074258mirror.0trial0262026-02-22 21:44:02cluster2cfa81703-817a-4074-adbc-5e6e200790bfup+stoppedlocal image is primary2026-02-22 21:44:03' 2026-02-22T21:44:12.386 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local last_update state 2026-02-22T21:44:12.386 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:44:12.388 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ last_update='2026-02-22 21:44:02' 2026-02-22T21:44:12.388 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:44:12.389 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ state=up+replaying 2026-02-22T21:44:12.390 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ echo up+replaying 2026-02-22T21:44:12.390 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep up+replaying 2026-02-22T21:44:12.390 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:up+replaying 2026-02-22T21:44:12.391 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:44:12.392 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:44:12.392 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ snap_name=snap8 2026-02-22T21:44:12.392 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_snap cluster2 mirror test snap8 2026-02-22T21:44:12.392 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:44:12.392 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:44:12.392 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:44:12.392 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap8 2026-02-22T21:44:12.392 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap8 --debug-rbd=20 --debug-journaler=20 2026-02-22T21:44:28.338 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.0 2026-02-22T21:44:28.339 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Sent signal 15 2026-02-22T21:44:28.339 INFO:tasks.rbd_mirror_thrash:waiting for 3.4437162018868017 secs before reviving daemons 2026-02-22T21:44:28.340 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial026.stderr:2026-02-22T21:44:28.338+0000 7f7e96d81640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 (PID: 47962) UID: 1000 2026-02-22T21:44:31.783 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.0 2026-02-22T21:44:31.784 INFO:teuthology.orchestra.run:waiting for 600 2026-02-22T21:44:31.784 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.0 2026-02-22T21:44:31.784 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Restarting daemon 2026-02-22T21:44:31.784 DEBUG:teuthology.orchestra.run.trial026:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 2026-02-22T21:44:31.787 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Started 2026-02-22T21:44:31.788 INFO:tasks.rbd_mirror_thrash:waiting for 72.4929475588083 secs before thrashing 2026-02-22T21:44:42.105 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T21:44:42.106 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:44:42.106 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:44:42.106 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:44:42.106 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T21:44:42.106 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:44:42.106 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:44:42.106 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:44:42.106 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=started 2026-02-22T21:44:42.107 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T21:44:42.107 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:44:42.107 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T21:44:43.107 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T21:44:43.107 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:44:43.107 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:44:43.107 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:44:43.107 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=started 2026-02-22T21:44:43.108 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_result 2026-02-22T21:44:43.108 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=stopped 2026-02-22T21:44:43.108 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:44:43.108 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_instance=cluster1 2026-02-22T21:44:43.108 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:44:43.108 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T21:44:43.108 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance=cluster1 2026-02-22T21:44:43.108 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local loop_instance 2026-02-22T21:44:43.108 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:44:43.108 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sleep 0 2026-02-22T21:44:43.108 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ grep -i state 2026-02-22T21:44:43.111 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:44:43.111 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ seq 0 3 2026-02-22T21:44:43.111 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:44:43.111 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:44:43.112 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:0 2026-02-22T21:44:43.112 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:44:43.112 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:44:43.112 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:44:43.112 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:0 2026-02-22T21:44:43.112 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:44:43.112 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:44:43.112 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:44:43.112 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=0 2026-02-22T21:44:43.112 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:44:43.112 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:44:43.112 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:44:43.113 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=0 2026-02-22T21:44:43.113 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T21:44:43.113 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:44:43.113 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:44:43.113 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:44:43.113 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:44:43.113 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:44:43.113 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:0 2026-02-22T21:44:43.113 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:44:43.113 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:44:43.113 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:44:43.113 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T21:44:43.114 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:44:43.114 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:44:43.114 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:44:43.114 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=0 2026-02-22T21:44:43.114 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=0 2026-02-22T21:44:43.114 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:44:43.122 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:44:43.122 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:44:43.122 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:44:43.122 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:44:43.163 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T21:44:43.163 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:44:43.163 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T21:44:43.166 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:44:43.166 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-22T21:44:43.166 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:1 2026-02-22T21:44:43.166 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:44:43.166 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:44:43.166 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T21:44:43.166 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:1 2026-02-22T21:44:43.166 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:44:43.166 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:44:43.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:44:43.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=1 2026-02-22T21:44:43.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T21:44:43.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:44:43.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:44:43.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=1 2026-02-22T21:44:43.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T21:44:43.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T21:44:43.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:1 2026-02-22T21:44:43.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:44:43.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:44:43.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T21:44:43.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:1 2026-02-22T21:44:43.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:44:43.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:44:43.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:44:43.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T21:44:43.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T21:44:43.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:44:43.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:44:43.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=1 2026-02-22T21:44:43.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=1 2026-02-22T21:44:43.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T21:44:43.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:44:43.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:44:43.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:44:43.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-22T21:44:43.219 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T21:44:43.219 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:44:43.219 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T21:44:43.222 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:44:43.222 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2026-02-22T21:44:43.222 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:2 2026-02-22T21:44:43.222 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:44:43.222 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:44:43.222 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T21:44:43.222 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:2 2026-02-22T21:44:43.222 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:44:43.222 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:44:43.223 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:44:43.223 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=2 2026-02-22T21:44:43.223 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T21:44:43.223 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:44:43.223 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:44:43.223 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=2 2026-02-22T21:44:43.223 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T21:44:43.223 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T21:44:43.223 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:2 2026-02-22T21:44:43.223 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:44:43.223 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:44:43.223 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T21:44:43.224 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:2 2026-02-22T21:44:43.224 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:44:43.224 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:44:43.224 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:44:43.224 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T21:44:43.224 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T21:44:43.224 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:44:43.224 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:44:43.224 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=2 2026-02-22T21:44:43.224 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=2 2026-02-22T21:44:43.224 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T21:44:43.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:44:43.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:44:43.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:44:43.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2026-02-22T21:44:43.273 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T21:44:43.273 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:44:43.274 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T21:44:43.277 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:44:43.277 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2026-02-22T21:44:43.277 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:3 2026-02-22T21:44:43.277 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:44:43.277 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:44:43.277 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T21:44:43.277 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:3 2026-02-22T21:44:43.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:44:43.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:44:43.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:44:43.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=3 2026-02-22T21:44:43.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T21:44:43.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:44:43.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:44:43.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=3 2026-02-22T21:44:43.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=3 2026-02-22T21:44:43.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T21:44:43.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:3 2026-02-22T21:44:43.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:44:43.279 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:44:43.279 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T21:44:43.279 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:3 2026-02-22T21:44:43.279 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:44:43.279 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:44:43.279 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:44:43.279 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=3 2026-02-22T21:44:43.279 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T21:44:43.279 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:44:43.279 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:44:43.279 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=3 2026-02-22T21:44:43.279 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=3 2026-02-22T21:44:43.280 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T21:44:43.286 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:44:43.287 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:44:43.287 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:44:43.287 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2026-02-22T21:44:43.327 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T21:44:43.327 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:44:43.327 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T21:44:43.328 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:44:43.328 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sleep 1 2026-02-22T21:44:44.329 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:44:44.329 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ seq 0 3 2026-02-22T21:44:44.332 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:44:44.332 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:44:44.333 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:0 2026-02-22T21:44:44.333 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:44:44.333 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:44:44.333 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:44:44.333 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:0 2026-02-22T21:44:44.333 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:44:44.333 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:44:44.333 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:44:44.333 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=0 2026-02-22T21:44:44.333 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:44:44.333 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:44:44.333 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:44:44.334 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=0 2026-02-22T21:44:44.334 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T21:44:44.334 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:44:44.334 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:44:44.334 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:44:44.334 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:44:44.334 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:44:44.334 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:0 2026-02-22T21:44:44.334 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:44:44.334 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:44:44.334 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:44:44.334 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T21:44:44.335 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:44:44.335 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:44:44.335 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:44:44.335 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=0 2026-02-22T21:44:44.335 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=0 2026-02-22T21:44:44.335 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:44:44.343 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:44:44.343 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:44:44.343 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:44:44.343 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:44:44.385 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T21:44:44.385 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:44:44.385 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T21:44:44.387 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:44:44.387 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-22T21:44:44.387 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:1 2026-02-22T21:44:44.387 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:44:44.387 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:44:44.387 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T21:44:44.387 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:1 2026-02-22T21:44:44.387 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:44:44.387 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:44:44.387 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:44:44.388 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=1 2026-02-22T21:44:44.388 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T21:44:44.388 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:44:44.388 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:44:44.388 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=1 2026-02-22T21:44:44.388 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T21:44:44.388 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T21:44:44.388 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:1 2026-02-22T21:44:44.388 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:44:44.388 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:44:44.388 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T21:44:44.388 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:1 2026-02-22T21:44:44.388 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:44:44.388 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:44:44.388 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:44:44.388 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T21:44:44.388 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T21:44:44.388 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:44:44.388 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:44:44.388 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=1 2026-02-22T21:44:44.388 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=1 2026-02-22T21:44:44.388 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T21:44:44.398 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:44:44.398 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:44:44.399 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:44:44.399 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-22T21:44:44.439 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T21:44:44.439 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:44:44.439 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T21:44:44.441 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:44:44.441 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2026-02-22T21:44:44.441 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:2 2026-02-22T21:44:44.441 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:44:44.441 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:44:44.441 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T21:44:44.441 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:2 2026-02-22T21:44:44.441 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:44:44.441 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:44:44.441 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:44:44.441 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=2 2026-02-22T21:44:44.442 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T21:44:44.442 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:44:44.442 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:44:44.442 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=2 2026-02-22T21:44:44.442 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T21:44:44.442 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T21:44:44.442 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:2 2026-02-22T21:44:44.442 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:44:44.442 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:44:44.442 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T21:44:44.442 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:2 2026-02-22T21:44:44.442 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:44:44.442 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:44:44.442 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:44:44.442 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T21:44:44.442 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T21:44:44.442 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:44:44.442 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:44:44.442 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=2 2026-02-22T21:44:44.442 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=2 2026-02-22T21:44:44.442 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T21:44:44.453 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:44:44.453 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:44:44.453 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:44:44.453 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2026-02-22T21:44:44.494 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T21:44:44.494 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:44:44.494 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T21:44:44.497 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:44:44.497 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2026-02-22T21:44:44.497 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:3 2026-02-22T21:44:44.497 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:44:44.497 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:44:44.497 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T21:44:44.497 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:3 2026-02-22T21:44:44.497 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:44:44.498 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:44:44.498 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:44:44.498 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=3 2026-02-22T21:44:44.498 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T21:44:44.498 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:44:44.498 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:44:44.498 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=3 2026-02-22T21:44:44.498 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=3 2026-02-22T21:44:44.498 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T21:44:44.498 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:3 2026-02-22T21:44:44.498 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:44:44.498 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:44:44.499 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T21:44:44.499 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:3 2026-02-22T21:44:44.499 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:44:44.499 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:44:44.499 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:44:44.499 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=3 2026-02-22T21:44:44.499 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T21:44:44.499 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:44:44.499 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:44:44.499 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=3 2026-02-22T21:44:44.499 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=3 2026-02-22T21:44:44.499 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T21:44:44.506 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:44:44.506 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:44:44.506 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:44:44.506 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2026-02-22T21:44:44.547 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T21:44:44.547 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:44:44.547 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T21:44:44.548 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:44:44.548 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sleep 2 2026-02-22T21:44:46.549 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:44:46.550 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ seq 0 3 2026-02-22T21:44:46.553 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:44:46.553 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:44:46.553 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:0 2026-02-22T21:44:46.553 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:44:46.553 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:44:46.553 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:44:46.553 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:0 2026-02-22T21:44:46.553 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:44:46.553 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:44:46.554 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:44:46.554 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=0 2026-02-22T21:44:46.554 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:44:46.554 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:44:46.554 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:44:46.554 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=0 2026-02-22T21:44:46.554 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T21:44:46.554 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:44:46.554 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:44:46.554 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:44:46.554 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:44:46.555 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:44:46.555 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:0 2026-02-22T21:44:46.555 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:44:46.555 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:44:46.555 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:44:46.555 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T21:44:46.555 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:44:46.555 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:44:46.555 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:44:46.555 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=0 2026-02-22T21:44:46.555 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=0 2026-02-22T21:44:46.556 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:44:46.565 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:44:46.565 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:44:46.565 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:44:46.565 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:44:46.605 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T21:44:46.606 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:44:46.606 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T21:44:46.609 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:44:46.609 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-22T21:44:46.609 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:1 2026-02-22T21:44:46.609 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:44:46.609 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:44:46.610 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T21:44:46.610 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:1 2026-02-22T21:44:46.610 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:44:46.610 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:44:46.610 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:44:46.610 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=1 2026-02-22T21:44:46.610 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T21:44:46.610 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:44:46.610 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:44:46.610 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=1 2026-02-22T21:44:46.610 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T21:44:46.611 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T21:44:46.611 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:1 2026-02-22T21:44:46.611 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:44:46.611 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:44:46.611 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T21:44:46.611 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:1 2026-02-22T21:44:46.611 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:44:46.611 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:44:46.611 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:44:46.611 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T21:44:46.611 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T21:44:46.611 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:44:46.611 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:44:46.612 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=1 2026-02-22T21:44:46.612 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=1 2026-02-22T21:44:46.612 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T21:44:46.619 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:44:46.619 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:44:46.619 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:44:46.619 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-22T21:44:46.659 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T21:44:46.659 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:44:46.659 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T21:44:46.662 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:44:46.662 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2026-02-22T21:44:46.662 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:2 2026-02-22T21:44:46.662 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:44:46.663 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:44:46.663 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T21:44:46.663 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:2 2026-02-22T21:44:46.663 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:44:46.663 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:44:46.663 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:44:46.663 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=2 2026-02-22T21:44:46.663 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T21:44:46.663 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:44:46.663 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:44:46.663 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=2 2026-02-22T21:44:46.664 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T21:44:46.664 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T21:44:46.664 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:2 2026-02-22T21:44:46.664 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:44:46.664 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:44:46.664 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T21:44:46.664 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:2 2026-02-22T21:44:46.664 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:44:46.664 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:44:46.664 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:44:46.664 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T21:44:46.664 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T21:44:46.665 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:44:46.665 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:44:46.665 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=2 2026-02-22T21:44:46.665 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=2 2026-02-22T21:44:46.665 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T21:44:46.672 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:44:46.672 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:44:46.672 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:44:46.672 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2026-02-22T21:44:46.712 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T21:44:46.712 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:44:46.713 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T21:44:46.714 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:44:46.715 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2026-02-22T21:44:46.715 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:3 2026-02-22T21:44:46.715 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:44:46.715 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:44:46.715 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T21:44:46.715 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:3 2026-02-22T21:44:46.715 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:44:46.715 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:44:46.715 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:44:46.715 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=3 2026-02-22T21:44:46.715 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T21:44:46.715 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:44:46.715 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:44:46.715 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=3 2026-02-22T21:44:46.715 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=3 2026-02-22T21:44:46.715 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T21:44:46.715 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:3 2026-02-22T21:44:46.715 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:44:46.715 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:44:46.715 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T21:44:46.715 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:3 2026-02-22T21:44:46.715 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:44:46.715 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:44:46.715 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:44:46.715 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=3 2026-02-22T21:44:46.716 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T21:44:46.716 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:44:46.716 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:44:46.716 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=3 2026-02-22T21:44:46.716 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=3 2026-02-22T21:44:46.716 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T21:44:46.726 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:44:46.726 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:44:46.726 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:44:46.726 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2026-02-22T21:44:46.767 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T21:44:46.767 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:44:46.768 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T21:44:46.769 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:44:46.769 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sleep 4 2026-02-22T21:44:50.770 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:44:50.770 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ seq 0 3 2026-02-22T21:44:50.773 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:44:50.773 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:44:50.774 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:0 2026-02-22T21:44:50.774 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:44:50.774 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:44:50.774 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:44:50.774 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:0 2026-02-22T21:44:50.774 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:44:50.775 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:44:50.775 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:44:50.775 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=0 2026-02-22T21:44:50.775 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:44:50.775 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:44:50.775 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:44:50.775 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=0 2026-02-22T21:44:50.775 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T21:44:50.776 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:44:50.776 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:44:50.776 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:44:50.776 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:44:50.776 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:44:50.776 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:0 2026-02-22T21:44:50.776 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:44:50.776 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:44:50.776 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:44:50.776 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T21:44:50.776 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:44:50.776 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:44:50.776 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:44:50.777 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=0 2026-02-22T21:44:50.777 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=0 2026-02-22T21:44:50.777 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:44:50.787 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:44:50.787 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:44:50.787 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:44:50.788 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:44:50.829 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T21:44:50.829 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:44:50.829 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T21:44:50.832 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:44:50.832 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-22T21:44:50.832 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:1 2026-02-22T21:44:50.832 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:44:50.832 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:44:50.832 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T21:44:50.832 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:1 2026-02-22T21:44:50.833 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:44:50.833 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:44:50.833 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:44:50.833 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=1 2026-02-22T21:44:50.833 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T21:44:50.833 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:44:50.833 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:44:50.833 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=1 2026-02-22T21:44:50.833 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T21:44:50.833 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T21:44:50.833 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:1 2026-02-22T21:44:50.833 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:44:50.834 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:44:50.834 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T21:44:50.834 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:1 2026-02-22T21:44:50.834 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:44:50.834 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:44:50.834 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:44:50.834 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T21:44:50.834 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T21:44:50.834 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:44:50.834 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:44:50.834 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=1 2026-02-22T21:44:50.834 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=1 2026-02-22T21:44:50.834 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T21:44:50.843 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:44:50.843 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:44:50.843 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:44:50.843 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-22T21:44:50.882 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T21:44:50.883 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:44:50.883 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T21:44:50.885 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:44:50.886 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2026-02-22T21:44:50.886 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:2 2026-02-22T21:44:50.886 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:44:50.886 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:44:50.886 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T21:44:50.886 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:2 2026-02-22T21:44:50.886 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:44:50.886 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:44:50.886 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:44:50.886 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=2 2026-02-22T21:44:50.887 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T21:44:50.887 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:44:50.887 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:44:50.887 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=2 2026-02-22T21:44:50.887 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T21:44:50.887 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T21:44:50.887 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:2 2026-02-22T21:44:50.887 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:44:50.887 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:44:50.887 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T21:44:50.887 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:2 2026-02-22T21:44:50.887 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:44:50.888 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:44:50.888 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:44:50.888 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T21:44:50.888 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T21:44:50.888 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:44:50.888 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:44:50.888 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=2 2026-02-22T21:44:50.888 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=2 2026-02-22T21:44:50.888 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T21:44:50.897 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:44:50.897 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:44:50.897 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:44:50.897 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2026-02-22T21:44:50.936 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T21:44:50.937 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:44:50.937 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T21:44:50.939 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:44:50.939 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2026-02-22T21:44:50.940 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:3 2026-02-22T21:44:50.940 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:44:50.940 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:44:50.940 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T21:44:50.940 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:3 2026-02-22T21:44:50.940 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:44:50.940 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:44:50.940 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:44:50.940 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=3 2026-02-22T21:44:50.940 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T21:44:50.940 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:44:50.941 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:44:50.941 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=3 2026-02-22T21:44:50.941 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=3 2026-02-22T21:44:50.941 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T21:44:50.941 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:3 2026-02-22T21:44:50.941 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:44:50.941 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:44:50.941 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T21:44:50.941 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:3 2026-02-22T21:44:50.941 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:44:50.941 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:44:50.942 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:44:50.942 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=3 2026-02-22T21:44:50.942 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T21:44:50.942 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:44:50.942 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:44:50.942 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=3 2026-02-22T21:44:50.942 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=3 2026-02-22T21:44:50.942 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T21:44:50.950 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:44:50.950 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:44:50.950 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:44:50.950 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2026-02-22T21:44:50.990 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T21:44:50.990 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:44:50.991 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T21:44:50.991 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:44:50.991 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sleep 8 2026-02-22T21:44:58.992 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:44:58.993 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ seq 0 3 2026-02-22T21:44:58.996 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:44:58.996 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:44:58.996 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:0 2026-02-22T21:44:58.996 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:44:58.996 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:44:58.996 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:44:58.997 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:0 2026-02-22T21:44:58.997 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:44:58.997 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:44:58.997 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:44:58.997 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=0 2026-02-22T21:44:58.997 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:44:58.998 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:44:58.998 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:44:58.998 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=0 2026-02-22T21:44:58.998 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T21:44:58.998 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:44:58.998 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:44:58.999 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:44:58.999 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:44:58.999 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:44:58.999 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:0 2026-02-22T21:44:58.999 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:44:58.999 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:44:59.000 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:44:59.000 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T21:44:59.000 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:44:59.000 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:44:59.000 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:44:59.000 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=0 2026-02-22T21:44:59.000 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=0 2026-02-22T21:44:59.001 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:44:59.009 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:44:59.009 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:44:59.009 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:44:59.009 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:44:59.051 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T21:44:59.051 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:44:59.051 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T21:44:59.054 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:44:59.054 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-22T21:44:59.055 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:1 2026-02-22T21:44:59.055 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:44:59.055 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:44:59.055 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T21:44:59.055 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:1 2026-02-22T21:44:59.055 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:44:59.055 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:44:59.055 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:44:59.055 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=1 2026-02-22T21:44:59.055 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T21:44:59.055 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:44:59.056 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:44:59.056 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=1 2026-02-22T21:44:59.056 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T21:44:59.056 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T21:44:59.056 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:1 2026-02-22T21:44:59.056 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:44:59.056 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:44:59.056 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T21:44:59.056 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:1 2026-02-22T21:44:59.056 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:44:59.056 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:44:59.056 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:44:59.057 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T21:44:59.057 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T21:44:59.057 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:44:59.057 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:44:59.057 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=1 2026-02-22T21:44:59.057 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=1 2026-02-22T21:44:59.057 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T21:44:59.065 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:44:59.065 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:44:59.065 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:44:59.065 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-22T21:44:59.106 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T21:44:59.106 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:44:59.106 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T21:44:59.111 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:44:59.111 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2026-02-22T21:44:59.111 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:2 2026-02-22T21:44:59.112 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:44:59.112 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:44:59.112 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T21:44:59.112 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:2 2026-02-22T21:44:59.112 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:44:59.112 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:44:59.112 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:44:59.112 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=2 2026-02-22T21:44:59.113 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T21:44:59.113 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:44:59.113 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:44:59.113 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=2 2026-02-22T21:44:59.113 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T21:44:59.113 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T21:44:59.113 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:2 2026-02-22T21:44:59.113 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:44:59.114 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:44:59.114 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T21:44:59.114 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:2 2026-02-22T21:44:59.114 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:44:59.114 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:44:59.114 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:44:59.114 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T21:44:59.114 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T21:44:59.115 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:44:59.115 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:44:59.115 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=2 2026-02-22T21:44:59.115 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=2 2026-02-22T21:44:59.115 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T21:44:59.120 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:44:59.120 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:44:59.120 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:44:59.121 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2026-02-22T21:44:59.160 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T21:44:59.160 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:44:59.160 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T21:44:59.161 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:44:59.161 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2026-02-22T21:44:59.161 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:3 2026-02-22T21:44:59.161 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:44:59.161 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:44:59.161 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T21:44:59.161 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:3 2026-02-22T21:44:59.161 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:44:59.161 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:44:59.162 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:44:59.162 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=3 2026-02-22T21:44:59.162 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T21:44:59.162 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:44:59.162 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:44:59.162 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=3 2026-02-22T21:44:59.162 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=3 2026-02-22T21:44:59.162 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T21:44:59.162 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:3 2026-02-22T21:44:59.162 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:44:59.162 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:44:59.162 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T21:44:59.162 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:3 2026-02-22T21:44:59.162 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:44:59.162 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:44:59.162 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:44:59.162 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=3 2026-02-22T21:44:59.162 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T21:44:59.162 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:44:59.162 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:44:59.162 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=3 2026-02-22T21:44:59.162 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=3 2026-02-22T21:44:59.162 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T21:44:59.173 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:44:59.173 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:44:59.173 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:44:59.173 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2026-02-22T21:44:59.214 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T21:44:59.214 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:44:59.214 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T21:44:59.215 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:44:59.215 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sleep 8 2026-02-22T21:45:05.938 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.2945159463520608) > weight (0.25) 2026-02-22T21:45:05.939 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.5596959021895044) > weight (0.25) 2026-02-22T21:45:05.939 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.5128138004442361) > weight (0.25) 2026-02-22T21:45:05.939 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.3 with skip (0.7983693955379995) > weight (0.25) 2026-02-22T21:45:05.939 INFO:tasks.rbd_mirror_thrash:waiting for 111.80166556712281 secs before thrashing 2026-02-22T21:45:07.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:45:07.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ seq 0 3 2026-02-22T21:45:07.222 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:45:07.222 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:45:07.222 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:0 2026-02-22T21:45:07.223 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:45:07.223 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:45:07.223 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:45:07.223 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:0 2026-02-22T21:45:07.223 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:45:07.223 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:45:07.224 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:45:07.224 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=0 2026-02-22T21:45:07.224 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:45:07.224 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:45:07.224 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:45:07.224 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=0 2026-02-22T21:45:07.225 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T21:45:07.225 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:45:07.225 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:45:07.225 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:45:07.225 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:45:07.226 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:45:07.226 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:0 2026-02-22T21:45:07.226 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:45:07.226 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:45:07.226 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:45:07.226 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T21:45:07.227 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:45:07.227 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:45:07.227 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:45:07.227 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=0 2026-02-22T21:45:07.227 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=0 2026-02-22T21:45:07.227 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:45:07.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:45:07.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:45:07.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:45:07.233 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:45:07.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T21:45:07.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:45:07.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T21:45:07.276 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:45:07.276 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-22T21:45:07.276 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:1 2026-02-22T21:45:07.276 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:45:07.276 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:45:07.276 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T21:45:07.276 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:1 2026-02-22T21:45:07.276 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:45:07.276 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:45:07.277 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:45:07.277 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=1 2026-02-22T21:45:07.277 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T21:45:07.277 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:45:07.277 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:45:07.277 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=1 2026-02-22T21:45:07.277 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T21:45:07.277 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T21:45:07.277 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:1 2026-02-22T21:45:07.277 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:45:07.277 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:45:07.277 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T21:45:07.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:1 2026-02-22T21:45:07.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:45:07.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:45:07.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:45:07.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T21:45:07.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T21:45:07.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:45:07.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:45:07.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=1 2026-02-22T21:45:07.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=1 2026-02-22T21:45:07.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T21:45:07.286 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:45:07.286 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:45:07.286 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:45:07.286 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-22T21:45:07.331 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ return 0 2026-02-22T21:45:07.332 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:45:07.332 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:45:07.332 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep -i Replaying 2026-02-22T21:45:07.333 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: "state": "Replaying" 2026-02-22T21:45:07.334 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_state=started 2026-02-22T21:45:07.334 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test started = started 2026-02-22T21:45:07.334 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:45:07.334 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2026-02-22T21:45:07.334 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T21:45:07.334 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:45:07.334 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:45:07.334 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:45:07.334 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = journal ']' 2026-02-22T21:45:07.334 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T21:45:07.334 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-22T21:45:07.334 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:45:07.334 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:45:07.334 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:45:07.334 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-22T21:45:09.121 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Snapshot ID: 26 2026-02-22T21:45:09.130 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test 2026-02-22T21:45:09.130 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T21:45:09.131 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:45:09.131 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:45:09.131 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:45:09.131 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster2-mirror-test.status 2026-02-22T21:45:09.131 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster2-mirror-test.status 2026-02-22T21:45:09.131 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T21:45:09.132 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_log=/tmp/tmp.j44LZPowal/cluster2-mirror-test.status 2026-02-22T21:45:09.133 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster1-mirror-test.status 2026-02-22T21:45:09.133 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster1-mirror-test.status 2026-02-22T21:45:09.133 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T21:45:09.134 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_status_log=/tmp/tmp.j44LZPowal/cluster1-mirror-test.status 2026-02-22T21:45:09.134 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.j44LZPowal/cluster2-mirror-test.status 2026-02-22T21:45:09.135 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:45:09.135 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:45:09.135 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:45:09.135 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster2-mirror-test.status 2026-02-22T21:45:09.135 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-22T21:45:09.135 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:45:09.166 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T21:45:09.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snapshot_id=26 2026-02-22T21:45:09.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:45:09.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:45:09.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.2 2026-02-22T21:45:09.369 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.j44LZPowal/cluster1-mirror-test.status 2026-02-22T21:45:09.370 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:45:09.370 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:45:09.370 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:45:09.370 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-test.status 2026-02-22T21:45:09.370 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T21:45:09.370 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:45:09.402 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:45:09.402 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:45:09.403 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T21:45:09.403 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T21:45:09.404 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T21:45:09.404 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T21:45:09.404 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 26 = '' 2026-02-22T21:45:09.404 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:45:09.404 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.4 2026-02-22T21:45:09.804 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.j44LZPowal/cluster1-mirror-test.status 2026-02-22T21:45:09.804 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:45:09.804 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:45:09.804 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:45:09.805 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-test.status 2026-02-22T21:45:09.805 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T21:45:09.805 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:45:09.834 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:45:09.835 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:45:09.836 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T21:45:09.836 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T21:45:09.836 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T21:45:09.836 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T21:45:09.836 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 26 = '' 2026-02-22T21:45:09.836 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:45:09.836 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.8 2026-02-22T21:45:10.637 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.j44LZPowal/cluster1-mirror-test.status 2026-02-22T21:45:10.637 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:45:10.638 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:45:10.638 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:45:10.638 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-test.status 2026-02-22T21:45:10.638 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T21:45:10.638 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:45:10.682 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:45:10.683 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:45:10.684 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T21:45:10.684 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T21:45:10.684 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T21:45:10.684 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T21:45:10.685 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 26 = '' 2026-02-22T21:45:10.685 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:45:10.685 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1.6 2026-02-22T21:45:12.286 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.j44LZPowal/cluster1-mirror-test.status 2026-02-22T21:45:12.286 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:45:12.286 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:45:12.286 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:45:12.286 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-test.status 2026-02-22T21:45:12.286 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T21:45:12.286 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:45:12.319 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:45:12.321 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id=26 2026-02-22T21:45:12.321 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 26 = 26 2026-02-22T21:45:12.321 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:45:12.321 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snap_present cluster1 mirror test snap8 2026-02-22T21:45:12.322 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:45:12.322 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:45:12.322 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:45:12.322 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap8 2026-02-22T21:45:12.322 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T21:45:12.322 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T21:45:12.322 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T21:45:13.322 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 info mirror/test@snap8 2026-02-22T21:45:13.350 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'test': 2026-02-22T21:45:13.350 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 512 MiB in 128 objects 2026-02-22T21:45:13.350 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T21:45:13.350 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 16 2026-02-22T21:45:13.350 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 10a25aa14179 2026-02-22T21:45:13.350 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.10a25aa14179 2026-02-22T21:45:13.350 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T21:45:13.351 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock, non-primary 2026-02-22T21:45:13.351 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T21:45:13.351 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T21:45:13.351 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:38:05 2026 2026-02-22T21:45:13.351 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:43:12 2026 2026-02-22T21:45:13.351 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:38:05 2026 2026-02-22T21:45:13.351 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: protected: False 2026-02-22T21:45:13.351 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T21:45:13.352 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T21:45:13.352 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: b410963b-cae9-4337-a6ef-da65321c9374 2026-02-22T21:45:13.352 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: false 2026-02-22T21:45:13.353 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:45:13.353 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n snap7-clean ']' 2026-02-22T21:45:13.353 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ compare_image_snaps mirror test snap7-clean 2026-02-22T21:45:13.353 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:45:13.353 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:45:13.354 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap7-clean 2026-02-22T21:45:13.354 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local ret=0 2026-02-22T21:45:13.354 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local rmt_export=/tmp/tmp.j44LZPowal/cluster2-mirror-test.export 2026-02-22T21:45:13.354 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local loc_export=/tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:45:13.354 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:45:13.354 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror export test@snap7-clean /tmp/tmp.j44LZPowal/cluster2-mirror-test.export 2026-02-22T21:45:13.691 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-22T21:45:13.701 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 -p mirror export test@snap7-clean /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:45:14.089 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-22T21:45:14.101 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ cmp /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:45:14.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:45:14.410 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:45:14.410 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ compare_image_snaps mirror test snap8 2026-02-22T21:45:14.410 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:45:14.410 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:45:14.411 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap8 2026-02-22T21:45:14.411 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local ret=0 2026-02-22T21:45:14.411 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local rmt_export=/tmp/tmp.j44LZPowal/cluster2-mirror-test.export 2026-02-22T21:45:14.411 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local loc_export=/tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:45:14.411 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:45:14.411 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror export test@snap8 /tmp/tmp.j44LZPowal/cluster2-mirror-test.export 2026-02-22T21:45:14.749 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-22T21:45:14.761 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 -p mirror export test@snap8 /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:45:15.146 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-22T21:45:15.160 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ cmp /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:45:15.345 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:45:15.469 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:45:15.469 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ clean_snap_name=snap8-clean 2026-02-22T21:45:15.470 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_snap cluster2 mirror test snap8-clean 2026-02-22T21:45:15.470 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:45:15.470 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:45:15.470 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:45:15.470 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap8-clean 2026-02-22T21:45:15.471 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap8-clean --debug-rbd=20 --debug-journaler=20 2026-02-22T21:45:16.128 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 10` 2026-02-22T21:45:16.128 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ stress_write_image cluster2 mirror test 2026-02-22T21:45:16.128 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:45:16.128 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:45:16.128 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:45:16.128 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-22T21:45:16.129 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local duration=12 2026-02-22T21:45:16.129 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set +e 2026-02-22T21:45:16.130 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ timeout 12s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2026-02-22T21:45:16.240 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:100 IOs, 26996736 bytes 2026-02-22T21:45:16.298 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:200 IOs, 41882624 bytes 2026-02-22T21:45:16.342 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:300 IOs, 76090368 bytes 2026-02-22T21:45:16.397 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:400 IOs, 102184960 bytes 2026-02-22T21:45:16.447 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:500 IOs, 142253056 bytes 2026-02-22T21:45:16.499 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:600 IOs, 193139712 bytes 2026-02-22T21:45:16.543 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:700 IOs, 217708544 bytes 2026-02-22T21:45:16.595 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:800 IOs, 262353920 bytes 2026-02-22T21:45:16.639 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:900 IOs, 292164608 bytes 2026-02-22T21:45:16.667 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1000 IOs, 292768768 bytes 2026-02-22T21:45:16.716 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1100 IOs, 345168896 bytes 2026-02-22T21:45:16.777 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1200 IOs, 420845568 bytes 2026-02-22T21:45:16.824 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1300 IOs, 457769984 bytes 2026-02-22T21:45:16.856 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1400 IOs, 471222272 bytes 2026-02-22T21:45:16.912 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1500 IOs, 552231936 bytes 2026-02-22T21:45:16.961 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1600 IOs, 589264896 bytes 2026-02-22T21:45:17.008 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1700 IOs, 619555840 bytes 2026-02-22T21:45:17.062 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1800 IOs, 677992448 bytes 2026-02-22T21:45:17.099 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1900 IOs, 697522176 bytes 2026-02-22T21:45:17.153 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2000 IOs, 738962432 bytes 2026-02-22T21:45:17.185 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2100 IOs, 747632640 bytes 2026-02-22T21:45:17.230 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2200 IOs, 787859456 bytes 2026-02-22T21:45:17.277 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2300 IOs, 828883968 bytes 2026-02-22T21:45:17.314 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2400 IOs, 840844288 bytes 2026-02-22T21:45:17.371 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2500 IOs, 889666560 bytes 2026-02-22T21:45:17.417 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2600 IOs, 927685632 bytes 2026-02-22T21:45:17.452 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2700 IOs, 937276416 bytes 2026-02-22T21:45:17.502 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2800 IOs, 988023808 bytes 2026-02-22T21:45:17.546 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2900 IOs, 1023830016 bytes 2026-02-22T21:45:17.589 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3000 IOs, 1047431168 bytes 2026-02-22T21:45:17.636 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3100 IOs, 1091421184 bytes 2026-02-22T21:45:17.675 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3200 IOs, 1115426816 bytes 2026-02-22T21:45:17.734 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3300 IOs, 1201618944 bytes 2026-02-22T21:45:17.806 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3400 IOs, 1293850624 bytes 2026-02-22T21:45:17.858 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3500 IOs, 1336921088 bytes 2026-02-22T21:45:17.916 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3600 IOs, 1398738944 bytes 2026-02-22T21:45:17.946 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3700 IOs, 1399345152 bytes 2026-02-22T21:45:17.980 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3800 IOs, 1419640832 bytes 2026-02-22T21:45:18.021 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3900 IOs, 1437691904 bytes 2026-02-22T21:45:18.054 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4000 IOs, 1445479424 bytes 2026-02-22T21:45:18.091 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4100 IOs, 1459065856 bytes 2026-02-22T21:45:18.157 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4200 IOs, 1537038336 bytes 2026-02-22T21:45:18.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4300 IOs, 1573118976 bytes 2026-02-22T21:45:18.234 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4400 IOs, 1582539776 bytes 2026-02-22T21:45:18.286 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4500 IOs, 1624715264 bytes 2026-02-22T21:45:18.335 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4600 IOs, 1677638656 bytes 2026-02-22T21:45:18.381 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4700 IOs, 1715690496 bytes 2026-02-22T21:45:18.433 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4800 IOs, 1758279680 bytes 2026-02-22T21:45:18.497 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4900 IOs, 1837075456 bytes 2026-02-22T21:45:18.543 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5000 IOs, 1874251776 bytes 2026-02-22T21:45:18.584 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5100 IOs, 1901110272 bytes 2026-02-22T21:45:18.618 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5200 IOs, 1908310016 bytes 2026-02-22T21:45:18.673 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5300 IOs, 1964233728 bytes 2026-02-22T21:45:18.718 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5400 IOs, 2004978688 bytes 2026-02-22T21:45:18.780 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5500 IOs, 2075448320 bytes 2026-02-22T21:45:18.826 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5600 IOs, 2124413952 bytes 2026-02-22T21:45:18.870 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5700 IOs, 2160031744 bytes 2026-02-22T21:45:18.920 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5800 IOs, 2246638592 bytes 2026-02-22T21:45:18.989 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5900 IOs, 2317731840 bytes 2026-02-22T21:45:19.043 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6000 IOs, 2343519232 bytes 2026-02-22T21:45:19.101 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6100 IOs, 2414130176 bytes 2026-02-22T21:45:19.152 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6200 IOs, 2487046144 bytes 2026-02-22T21:45:19.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6300 IOs, 2514580480 bytes 2026-02-22T21:45:19.254 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6400 IOs, 2572013568 bytes 2026-02-22T21:45:19.302 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6500 IOs, 2615844864 bytes 2026-02-22T21:45:19.345 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6600 IOs, 2620852224 bytes 2026-02-22T21:45:19.377 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6700 IOs, 2621466624 bytes 2026-02-22T21:45:19.428 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6800 IOs, 2679078912 bytes 2026-02-22T21:45:19.477 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6900 IOs, 2702981120 bytes 2026-02-22T21:45:19.533 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7000 IOs, 2766788608 bytes 2026-02-22T21:45:19.583 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7100 IOs, 2802236416 bytes 2026-02-22T21:45:19.616 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7200 IOs, 2805128192 bytes 2026-02-22T21:45:19.664 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7300 IOs, 2847062016 bytes 2026-02-22T21:45:19.732 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7400 IOs, 2928739328 bytes 2026-02-22T21:45:19.766 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7500 IOs, 2935145472 bytes 2026-02-22T21:45:19.797 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7600 IOs, 2935766016 bytes 2026-02-22T21:45:19.856 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7700 IOs, 2992494592 bytes 2026-02-22T21:45:19.922 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7800 IOs, 3086927872 bytes 2026-02-22T21:45:19.970 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7900 IOs, 3118290944 bytes 2026-02-22T21:45:20.009 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8000 IOs, 3153032192 bytes 2026-02-22T21:45:20.069 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8100 IOs, 3193112576 bytes 2026-02-22T21:45:20.098 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8200 IOs, 3201752064 bytes 2026-02-22T21:45:20.169 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8300 IOs, 3301096448 bytes 2026-02-22T21:45:20.226 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8400 IOs, 3334237184 bytes 2026-02-22T21:45:20.280 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8500 IOs, 3390954496 bytes 2026-02-22T21:45:20.328 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8600 IOs, 3436569600 bytes 2026-02-22T21:45:20.378 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8700 IOs, 3480677376 bytes 2026-02-22T21:45:20.417 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8800 IOs, 3494621184 bytes 2026-02-22T21:45:20.460 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8900 IOs, 3520941056 bytes 2026-02-22T21:45:20.517 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9000 IOs, 3584425984 bytes 2026-02-22T21:45:20.573 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9100 IOs, 3640637440 bytes 2026-02-22T21:45:20.619 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9200 IOs, 3689767936 bytes 2026-02-22T21:45:20.660 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9300 IOs, 3712310272 bytes 2026-02-22T21:45:20.710 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9400 IOs, 3754101760 bytes 2026-02-22T21:45:20.759 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9500 IOs, 3786637312 bytes 2026-02-22T21:45:20.825 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9600 IOs, 3864537088 bytes 2026-02-22T21:45:20.886 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9700 IOs, 3932804096 bytes 2026-02-22T21:45:20.925 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9800 IOs, 3946455040 bytes 2026-02-22T21:45:20.965 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9900 IOs, 3970157568 bytes 2026-02-22T21:45:21.017 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10000 IOs, 4007093248 bytes 2026-02-22T21:45:21.052 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10100 IOs, 4023768064 bytes 2026-02-22T21:45:21.098 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10200 IOs, 4040823808 bytes 2026-02-22T21:45:21.143 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10300 IOs, 4065392640 bytes 2026-02-22T21:45:21.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10400 IOs, 4129042432 bytes 2026-02-22T21:45:21.233 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10500 IOs, 4130811904 bytes 2026-02-22T21:45:21.283 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10600 IOs, 4173594624 bytes 2026-02-22T21:45:21.332 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10700 IOs, 4220731392 bytes 2026-02-22T21:45:21.374 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10800 IOs, 4237087744 bytes 2026-02-22T21:45:21.447 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10900 IOs, 4324048896 bytes 2026-02-22T21:45:21.517 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11000 IOs, 4412642304 bytes 2026-02-22T21:45:21.568 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11100 IOs, 4476565504 bytes 2026-02-22T21:45:21.629 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11200 IOs, 4523293696 bytes 2026-02-22T21:45:21.669 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11300 IOs, 4539265024 bytes 2026-02-22T21:45:21.709 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11400 IOs, 4563581952 bytes 2026-02-22T21:45:21.760 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11500 IOs, 4605753344 bytes 2026-02-22T21:45:21.822 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11600 IOs, 4670352384 bytes 2026-02-22T21:45:21.885 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11700 IOs, 4749673472 bytes 2026-02-22T21:45:21.950 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11800 IOs, 4820018176 bytes 2026-02-22T21:45:21.995 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11900 IOs, 4873348096 bytes 2026-02-22T21:45:22.039 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12000 IOs, 4885926912 bytes 2026-02-22T21:45:22.086 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12100 IOs, 4912966656 bytes 2026-02-22T21:45:22.123 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12200 IOs, 4937246720 bytes 2026-02-22T21:45:22.188 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12300 IOs, 4998639616 bytes 2026-02-22T21:45:22.223 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12400 IOs, 5007237120 bytes 2026-02-22T21:45:22.275 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12500 IOs, 5047354368 bytes 2026-02-22T21:45:22.326 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12600 IOs, 5084283904 bytes 2026-02-22T21:45:22.396 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12700 IOs, 5172992000 bytes 2026-02-22T21:45:22.446 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12800 IOs, 5203240960 bytes 2026-02-22T21:45:22.501 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12900 IOs, 5260144640 bytes 2026-02-22T21:45:22.565 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13000 IOs, 5337308160 bytes 2026-02-22T21:45:22.611 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13100 IOs, 5364342784 bytes 2026-02-22T21:45:22.643 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13200 IOs, 5370706944 bytes 2026-02-22T21:45:22.695 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13300 IOs, 5418540032 bytes 2026-02-22T21:45:22.774 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13400 IOs, 5529879552 bytes 2026-02-22T21:45:22.824 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13500 IOs, 5570780160 bytes 2026-02-22T21:45:22.890 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13600 IOs, 5634688000 bytes 2026-02-22T21:45:22.948 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13700 IOs, 5695851520 bytes 2026-02-22T21:45:22.982 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13800 IOs, 5700701184 bytes 2026-02-22T21:45:23.035 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13900 IOs, 5758693376 bytes 2026-02-22T21:45:23.090 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14000 IOs, 5806351360 bytes 2026-02-22T21:45:23.133 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14100 IOs, 5838806016 bytes 2026-02-22T21:45:23.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14200 IOs, 5898330112 bytes 2026-02-22T21:45:23.226 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14300 IOs, 5898944512 bytes 2026-02-22T21:45:23.289 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14400 IOs, 5973775360 bytes 2026-02-22T21:45:23.340 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14500 IOs, 6014013440 bytes 2026-02-22T21:45:23.394 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14600 IOs, 6062156800 bytes 2026-02-22T21:45:23.444 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14700 IOs, 6092223488 bytes 2026-02-22T21:45:23.494 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14800 IOs, 6138999808 bytes 2026-02-22T21:45:23.545 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14900 IOs, 6189948928 bytes 2026-02-22T21:45:23.589 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15000 IOs, 6221474816 bytes 2026-02-22T21:45:23.657 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15100 IOs, 6283510784 bytes 2026-02-22T21:45:23.714 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15200 IOs, 6338769920 bytes 2026-02-22T21:45:23.770 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15300 IOs, 6409246720 bytes 2026-02-22T21:45:23.813 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15400 IOs, 6414665728 bytes 2026-02-22T21:45:23.843 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15500 IOs, 6416886784 bytes 2026-02-22T21:45:23.888 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15600 IOs, 6456225792 bytes 2026-02-22T21:45:23.939 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15700 IOs, 6495794176 bytes 2026-02-22T21:45:23.978 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15800 IOs, 6513978368 bytes 2026-02-22T21:45:24.009 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15900 IOs, 6518521856 bytes 2026-02-22T21:45:24.056 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16000 IOs, 6559814656 bytes 2026-02-22T21:45:24.101 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16100 IOs, 6596026368 bytes 2026-02-22T21:45:24.149 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16200 IOs, 6638767104 bytes 2026-02-22T21:45:24.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16300 IOs, 6676378624 bytes 2026-02-22T21:45:24.238 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16400 IOs, 6684595200 bytes 2026-02-22T21:45:24.276 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16500 IOs, 6699949056 bytes 2026-02-22T21:45:24.326 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16600 IOs, 6742240256 bytes 2026-02-22T21:45:24.379 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16700 IOs, 6782683136 bytes 2026-02-22T21:45:24.434 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16800 IOs, 6837300224 bytes 2026-02-22T21:45:24.486 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16900 IOs, 6870728704 bytes 2026-02-22T21:45:24.546 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17000 IOs, 6939379712 bytes 2026-02-22T21:45:24.602 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17100 IOs, 6982855680 bytes 2026-02-22T21:45:24.666 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17200 IOs, 7068481536 bytes 2026-02-22T21:45:24.701 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17300 IOs, 7077923840 bytes 2026-02-22T21:45:24.740 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17400 IOs, 7093990400 bytes 2026-02-22T21:45:24.788 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17500 IOs, 7129691136 bytes 2026-02-22T21:45:24.821 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17600 IOs, 7131509760 bytes 2026-02-22T21:45:24.862 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17700 IOs, 7154096128 bytes 2026-02-22T21:45:24.906 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17800 IOs, 7186765824 bytes 2026-02-22T21:45:24.951 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17900 IOs, 7216555008 bytes 2026-02-22T21:45:25.021 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18000 IOs, 7299974144 bytes 2026-02-22T21:45:25.053 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18100 IOs, 7303814144 bytes 2026-02-22T21:45:25.109 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18200 IOs, 7363003392 bytes 2026-02-22T21:45:25.153 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18300 IOs, 7383930880 bytes 2026-02-22T21:45:25.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18400 IOs, 7424415744 bytes 2026-02-22T21:45:25.295 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18500 IOs, 7549507584 bytes 2026-02-22T21:45:25.338 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18600 IOs, 7570191360 bytes 2026-02-22T21:45:25.380 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18700 IOs, 7587057664 bytes 2026-02-22T21:45:25.429 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18800 IOs, 7628820480 bytes 2026-02-22T21:45:25.482 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18900 IOs, 7679513600 bytes 2026-02-22T21:45:25.528 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19000 IOs, 7702166528 bytes 2026-02-22T21:45:25.577 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19100 IOs, 7752287232 bytes 2026-02-22T21:45:25.627 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19200 IOs, 7773747200 bytes 2026-02-22T21:45:25.669 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19300 IOs, 7793787904 bytes 2026-02-22T21:45:25.723 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19400 IOs, 7831529472 bytes 2026-02-22T21:45:25.784 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19500 IOs, 7905064960 bytes 2026-02-22T21:45:25.842 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19600 IOs, 7955789824 bytes 2026-02-22T21:45:25.903 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19700 IOs, 8022214656 bytes 2026-02-22T21:45:26.136 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19800 IOs, 8047477760 bytes 2026-02-22T21:45:26.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19900 IOs, 8092110848 bytes 2026-02-22T21:45:26.237 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20000 IOs, 8112382976 bytes 2026-02-22T21:45:26.294 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20100 IOs, 8176068608 bytes 2026-02-22T21:45:26.333 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20200 IOs, 8195474432 bytes 2026-02-22T21:45:26.375 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20300 IOs, 8211566592 bytes 2026-02-22T21:45:26.444 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20400 IOs, 8280365056 bytes 2026-02-22T21:45:26.503 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20500 IOs, 8338450432 bytes 2026-02-22T21:45:26.556 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20600 IOs, 8377643008 bytes 2026-02-22T21:45:26.613 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20700 IOs, 8436511744 bytes 2026-02-22T21:45:26.675 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20800 IOs, 8497815552 bytes 2026-02-22T21:45:26.709 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20900 IOs, 8498407424 bytes 2026-02-22T21:45:26.765 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21000 IOs, 8552223744 bytes 2026-02-22T21:45:26.815 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21100 IOs, 8597769216 bytes 2026-02-22T21:45:26.859 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21200 IOs, 8618552320 bytes 2026-02-22T21:45:26.891 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21300 IOs, 8619168768 bytes 2026-02-22T21:45:26.932 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21400 IOs, 8645152768 bytes 2026-02-22T21:45:26.994 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21500 IOs, 8729939968 bytes 2026-02-22T21:45:27.033 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21600 IOs, 8743567360 bytes 2026-02-22T21:45:27.092 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21700 IOs, 8806039552 bytes 2026-02-22T21:45:27.137 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21800 IOs, 8837650432 bytes 2026-02-22T21:45:27.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21900 IOs, 8877169664 bytes 2026-02-22T21:45:27.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22000 IOs, 8946656256 bytes 2026-02-22T21:45:27.294 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22100 IOs, 8963544064 bytes 2026-02-22T21:45:27.343 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22200 IOs, 9014753280 bytes 2026-02-22T21:45:27.406 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22300 IOs, 9058755584 bytes 2026-02-22T21:45:27.439 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22400 IOs, 9059379200 bytes 2026-02-22T21:45:27.473 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22500 IOs, 9078680576 bytes 2026-02-22T21:45:27.519 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22600 IOs, 9103521792 bytes 2026-02-22T21:45:27.568 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22700 IOs, 9150239744 bytes 2026-02-22T21:45:27.612 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22800 IOs, 9177165824 bytes 2026-02-22T21:45:27.664 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22900 IOs, 9228190720 bytes 2026-02-22T21:45:27.717 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23000 IOs, 9277318144 bytes 2026-02-22T21:45:27.774 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23100 IOs, 9341542400 bytes 2026-02-22T21:45:27.831 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23200 IOs, 9402459136 bytes 2026-02-22T21:45:27.887 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23300 IOs, 9454538752 bytes 2026-02-22T21:45:27.937 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23400 IOs, 9484504064 bytes 2026-02-22T21:45:27.986 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23500 IOs, 9518245888 bytes 2026-02-22T21:45:28.015 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23600 IOs, 9518862336 bytes 2026-02-22T21:45:28.063 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23700 IOs, 9576463360 bytes 2026-02-22T21:45:28.105 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23800 IOs, 9577080832 bytes 2026-02-22T21:45:28.134 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ error_code=124 2026-02-22T21:45:28.134 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set -e 2026-02-22T21:45:28.134 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' 124 -eq 124 ']' 2026-02-22T21:45:28.134 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:45:28.134 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T21:45:28.135 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:45:28.135 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:45:28.135 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:45:28.135 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state_pattern=up+replaying 2026-02-22T21:45:28.135 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local description_pattern= 2026-02-22T21:45:28.135 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local service_pattern= 2026-02-22T21:45:28.135 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:45:28.135 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T21:45:29.134 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T21:45:29.134 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:45:29.134 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:45:29.134 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:45:29.134 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state_pattern=up+replaying 2026-02-22T21:45:29.134 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local description_pattern= 2026-02-22T21:45:29.134 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local service_pattern= 2026-02-22T21:45:29.134 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status 2026-02-22T21:45:29.135 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ CEPH_ARGS= 2026-02-22T21:45:29.135 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:45:29.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ status='test: 2026-02-22T21:45:29.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: global_id: b410963b-cae9-4337-a6ef-da65321c9374 2026-02-22T21:45:29.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: state: up+replaying 2026-02-22T21:45:29.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771796636,"remote_snapshot_timestamp":1771796636,"replay_state":"idle"} 2026-02-22T21:45:29.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: service: mirror.1 on trial026 2026-02-22T21:45:29.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: last_update: 2026-02-22 21:45:02 2026-02-22T21:45:29.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: peer_sites: 2026-02-22T21:45:29.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: name: cluster2 2026-02-22T21:45:29.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: state: up+stopped 2026-02-22T21:45:29.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: description: local image is primary 2026-02-22T21:45:29.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: last_update: 2026-02-22 21:45:03' 2026-02-22T21:45:29.169 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T21:45:29.169 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: state: up+replaying 2026-02-22T21:45:29.169 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep '^ description: .*' 2026-02-22T21:45:29.170 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771796636,"remote_snapshot_timestamp":1771796636,"replay_state":"idle"} 2026-02-22T21:45:29.170 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n '' ']' 2026-02-22T21:45:29.170 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ echo up+replaying 2026-02-22T21:45:29.170 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep '^up+' 2026-02-22T21:45:29.171 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:up+replaying 2026-02-22T21:45:29.171 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:45:29.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: service: mirror.1 on trial026 2026-02-22T21:45:29.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local last_update 2026-02-22T21:45:29.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:45:29.174 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ last_update='2026-02-22 21:45:02' 2026-02-22T21:45:29.174 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 21:45:02' 2026-02-22T21:45:29.174 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:45:29.174 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:45:29.174 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:45:29.174 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state_pattern=up+replaying 2026-02-22T21:45:29.174 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local 'prev_last_update=2026-02-22 21:45:02' 2026-02-22T21:45:29.174 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status 2026-02-22T21:45:29.174 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ CEPH_ARGS= 2026-02-22T21:45:29.174 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:45:29.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ status='OKOKOK142104261mirror.1trial02619.2.3-689-gf90f4eaetrueOK43584361mirror.2trial02619.2.3-689-gf90f4eaefalseOK44434446mirror.3trial02619.2.3-689-gf90f4eaefalseOK45034506mirror.0trial02619.2.3-689-gf90f4eaefalseOKtestb410963b-cae9-4337-a6ef-da65321c9374up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771796636,"remote_snapshot_timestamp":1771796636,"replay_state":"idle"}42104261mirror.1trial0262026-02-22 21:45:02cluster2cfa81703-817a-4074-adbc-5e6e200790bfup+stoppedlocal image is primary2026-02-22 21:45:03' 2026-02-22T21:45:29.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local last_update state 2026-02-22T21:45:29.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:45:29.209 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ last_update='2026-02-22 21:45:02' 2026-02-22T21:45:29.209 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:45:29.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ state=up+replaying 2026-02-22T21:45:29.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ echo up+replaying 2026-02-22T21:45:29.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep up+replaying 2026-02-22T21:45:29.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:up+replaying 2026-02-22T21:45:29.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:45:29.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:45:29.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ snap_name=snap9 2026-02-22T21:45:29.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_snap cluster2 mirror test snap9 2026-02-22T21:45:29.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:45:29.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:45:29.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:45:29.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap9 2026-02-22T21:45:29.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap9 --debug-rbd=20 --debug-journaler=20 2026-02-22T21:45:44.281 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.4495607984835124) > weight (0.25) 2026-02-22T21:45:44.281 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.4613468386125752) > weight (0.25) 2026-02-22T21:45:44.281 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.7015162245108101) > weight (0.25) 2026-02-22T21:45:44.281 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.3 2026-02-22T21:45:44.281 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Sent signal 15 2026-02-22T21:45:44.281 INFO:tasks.rbd_mirror_thrash:waiting for 7.110654050865438 secs before reviving daemons 2026-02-22T21:45:44.282 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial026.stderr:2026-02-22T21:45:44.281+0000 7f4bbd37d640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.3 (PID: 52842) UID: 1000 2026-02-22T21:45:51.393 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.3 2026-02-22T21:45:51.393 INFO:teuthology.orchestra.run:waiting for 600 2026-02-22T21:45:51.393 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.3 2026-02-22T21:45:51.394 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Restarting daemon 2026-02-22T21:45:51.394 DEBUG:teuthology.orchestra.run.trial026:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.3 2026-02-22T21:45:51.396 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Started 2026-02-22T21:45:51.396 INFO:tasks.rbd_mirror_thrash:waiting for 89.02912335812675 secs before thrashing 2026-02-22T21:46:00.067 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T21:46:00.067 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:46:00.067 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:46:00.068 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:46:00.068 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T21:46:00.068 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:46:00.068 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:46:00.068 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:46:00.068 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=started 2026-02-22T21:46:00.069 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T21:46:00.069 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:46:00.069 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T21:46:01.067 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T21:46:01.067 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:46:01.067 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:46:01.067 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:46:01.067 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=started 2026-02-22T21:46:01.067 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_result 2026-02-22T21:46:01.067 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=stopped 2026-02-22T21:46:01.068 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:46:01.068 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_instance=cluster1 2026-02-22T21:46:01.068 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ grep -i state 2026-02-22T21:46:01.068 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:46:01.068 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T21:46:01.068 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance=cluster1 2026-02-22T21:46:01.068 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local loop_instance 2026-02-22T21:46:01.068 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:46:01.068 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sleep 0 2026-02-22T21:46:01.068 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:46:01.069 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ seq 0 3 2026-02-22T21:46:01.070 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:46:01.070 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:46:01.070 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:0 2026-02-22T21:46:01.070 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:46:01.070 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:46:01.070 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:46:01.070 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:0 2026-02-22T21:46:01.070 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:46:01.070 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:46:01.070 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:46:01.070 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=0 2026-02-22T21:46:01.070 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:46:01.070 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:46:01.070 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:46:01.070 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=0 2026-02-22T21:46:01.070 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T21:46:01.070 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:46:01.070 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:46:01.070 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:46:01.070 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:46:01.071 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:46:01.071 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:0 2026-02-22T21:46:01.071 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:46:01.071 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:46:01.071 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:46:01.071 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T21:46:01.071 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:46:01.071 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:46:01.071 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:46:01.071 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=0 2026-02-22T21:46:01.071 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=0 2026-02-22T21:46:01.071 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:46:01.085 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:46:01.085 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:46:01.085 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:46:01.085 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:46:01.126 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T21:46:01.127 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:46:01.127 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T21:46:01.132 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:46:01.132 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-22T21:46:01.132 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:1 2026-02-22T21:46:01.133 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:46:01.133 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:46:01.133 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T21:46:01.133 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:1 2026-02-22T21:46:01.133 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:46:01.134 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:46:01.134 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:46:01.134 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=1 2026-02-22T21:46:01.134 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T21:46:01.134 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:46:01.134 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:46:01.135 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=1 2026-02-22T21:46:01.135 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T21:46:01.135 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T21:46:01.135 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:1 2026-02-22T21:46:01.135 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:46:01.135 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:46:01.136 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T21:46:01.136 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:1 2026-02-22T21:46:01.136 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:46:01.136 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:46:01.136 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:46:01.136 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T21:46:01.137 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T21:46:01.137 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:46:01.137 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:46:01.137 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=1 2026-02-22T21:46:01.137 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=1 2026-02-22T21:46:01.138 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T21:46:01.141 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:46:01.142 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:46:01.142 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:46:01.142 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-22T21:46:01.185 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ return 0 2026-02-22T21:46:01.185 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:46:01.186 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:46:01.186 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep -i Replaying 2026-02-22T21:46:01.186 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: "state": "Replaying" 2026-02-22T21:46:01.187 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_state=started 2026-02-22T21:46:01.187 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test started = started 2026-02-22T21:46:01.188 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:46:01.188 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2026-02-22T21:46:01.188 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T21:46:01.188 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:46:01.188 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:46:01.188 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:46:01.188 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = journal ']' 2026-02-22T21:46:01.188 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T21:46:01.188 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-22T21:46:01.188 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:46:01.188 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:46:01.188 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:46:01.188 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-22T21:46:03.066 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Snapshot ID: 29 2026-02-22T21:46:03.074 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test 2026-02-22T21:46:03.074 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T21:46:03.074 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:46:03.074 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:46:03.074 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:46:03.074 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster2-mirror-test.status 2026-02-22T21:46:03.074 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster2-mirror-test.status 2026-02-22T21:46:03.074 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T21:46:03.075 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_log=/tmp/tmp.j44LZPowal/cluster2-mirror-test.status 2026-02-22T21:46:03.075 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster1-mirror-test.status 2026-02-22T21:46:03.076 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster1-mirror-test.status 2026-02-22T21:46:03.076 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T21:46:03.077 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_status_log=/tmp/tmp.j44LZPowal/cluster1-mirror-test.status 2026-02-22T21:46:03.077 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.j44LZPowal/cluster2-mirror-test.status 2026-02-22T21:46:03.077 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:46:03.077 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:46:03.077 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:46:03.077 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster2-mirror-test.status 2026-02-22T21:46:03.077 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-22T21:46:03.077 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:46:03.110 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T21:46:03.112 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snapshot_id=29 2026-02-22T21:46:03.112 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:46:03.112 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:46:03.112 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.2 2026-02-22T21:46:03.314 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.j44LZPowal/cluster1-mirror-test.status 2026-02-22T21:46:03.314 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:46:03.314 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:46:03.314 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:46:03.314 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-test.status 2026-02-22T21:46:03.315 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T21:46:03.315 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:46:03.346 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:46:03.346 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:46:03.347 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T21:46:03.347 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T21:46:03.347 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T21:46:03.348 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T21:46:03.348 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 29 = '' 2026-02-22T21:46:03.348 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:46:03.348 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.4 2026-02-22T21:46:03.749 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.j44LZPowal/cluster1-mirror-test.status 2026-02-22T21:46:03.749 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:46:03.749 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:46:03.749 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:46:03.750 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-test.status 2026-02-22T21:46:03.750 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T21:46:03.750 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:46:03.781 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:46:03.781 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:46:03.782 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T21:46:03.783 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T21:46:03.783 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T21:46:03.783 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T21:46:03.783 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 29 = '' 2026-02-22T21:46:03.783 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:46:03.783 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.8 2026-02-22T21:46:04.583 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.j44LZPowal/cluster1-mirror-test.status 2026-02-22T21:46:04.583 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:46:04.583 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:46:04.583 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:46:04.584 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-test.status 2026-02-22T21:46:04.584 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T21:46:04.584 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:46:04.827 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:46:04.829 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id=29 2026-02-22T21:46:04.830 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 29 = 29 2026-02-22T21:46:04.830 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:46:04.830 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snap_present cluster1 mirror test snap9 2026-02-22T21:46:04.830 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:46:04.830 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:46:04.830 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:46:04.831 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap9 2026-02-22T21:46:04.831 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T21:46:04.831 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T21:46:04.831 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T21:46:05.830 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 info mirror/test@snap9 2026-02-22T21:46:05.860 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'test': 2026-02-22T21:46:05.860 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 512 MiB in 128 objects 2026-02-22T21:46:05.860 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T21:46:05.860 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 18 2026-02-22T21:46:05.860 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 10a25aa14179 2026-02-22T21:46:05.860 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.10a25aa14179 2026-02-22T21:46:05.860 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T21:46:05.860 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock, non-primary 2026-02-22T21:46:05.860 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T21:46:05.860 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T21:46:05.860 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:38:05 2026 2026-02-22T21:46:05.860 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:45:13 2026 2026-02-22T21:46:05.860 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:38:05 2026 2026-02-22T21:46:05.860 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: protected: False 2026-02-22T21:46:05.860 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T21:46:05.860 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T21:46:05.860 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: b410963b-cae9-4337-a6ef-da65321c9374 2026-02-22T21:46:05.860 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: false 2026-02-22T21:46:05.861 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:46:05.862 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n snap8-clean ']' 2026-02-22T21:46:05.862 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ compare_image_snaps mirror test snap8-clean 2026-02-22T21:46:05.862 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:46:05.862 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:46:05.862 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap8-clean 2026-02-22T21:46:05.862 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local ret=0 2026-02-22T21:46:05.862 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local rmt_export=/tmp/tmp.j44LZPowal/cluster2-mirror-test.export 2026-02-22T21:46:05.862 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local loc_export=/tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:46:05.862 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:46:05.862 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror export test@snap8-clean /tmp/tmp.j44LZPowal/cluster2-mirror-test.export 2026-02-22T21:46:06.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-22T21:46:06.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 -p mirror export test@snap8-clean /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:46:06.593 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-22T21:46:06.604 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ cmp /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:46:06.784 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:46:06.907 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:46:06.908 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ compare_image_snaps mirror test snap9 2026-02-22T21:46:06.908 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:46:06.908 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:46:06.908 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap9 2026-02-22T21:46:06.908 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local ret=0 2026-02-22T21:46:06.908 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local rmt_export=/tmp/tmp.j44LZPowal/cluster2-mirror-test.export 2026-02-22T21:46:06.908 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local loc_export=/tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:46:06.908 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:46:06.908 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror export test@snap9 /tmp/tmp.j44LZPowal/cluster2-mirror-test.export 2026-02-22T21:46:07.250 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-22T21:46:07.258 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 -p mirror export test@snap9 /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:46:07.625 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-22T21:46:07.638 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ cmp /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:46:07.824 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:46:07.950 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:46:07.951 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ clean_snap_name=snap9-clean 2026-02-22T21:46:07.951 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_snap cluster2 mirror test snap9-clean 2026-02-22T21:46:07.951 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:46:07.951 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:46:07.951 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:46:07.951 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap9-clean 2026-02-22T21:46:07.951 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap9-clean --debug-rbd=20 --debug-journaler=20 2026-02-22T21:46:08.075 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 10` 2026-02-22T21:46:08.075 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ stress_write_image cluster2 mirror test 2026-02-22T21:46:08.075 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:46:08.075 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:46:08.075 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:46:08.076 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-22T21:46:08.077 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local duration=13 2026-02-22T21:46:08.077 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set +e 2026-02-22T21:46:08.077 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ timeout 13s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2026-02-22T21:46:08.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:100 IOs, 26617856 bytes 2026-02-22T21:46:08.243 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:200 IOs, 39554048 bytes 2026-02-22T21:46:08.296 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:300 IOs, 72899584 bytes 2026-02-22T21:46:08.346 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:400 IOs, 111921152 bytes 2026-02-22T21:46:08.403 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:500 IOs, 176404480 bytes 2026-02-22T21:46:08.445 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:600 IOs, 191682560 bytes 2026-02-22T21:46:08.514 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:700 IOs, 270959616 bytes 2026-02-22T21:46:08.564 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:800 IOs, 311660544 bytes 2026-02-22T21:46:08.614 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:900 IOs, 354244608 bytes 2026-02-22T21:46:08.677 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1000 IOs, 410729472 bytes 2026-02-22T21:46:08.713 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1100 IOs, 423065600 bytes 2026-02-22T21:46:08.769 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1200 IOs, 484036608 bytes 2026-02-22T21:46:08.804 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1300 IOs, 484667392 bytes 2026-02-22T21:46:08.848 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1400 IOs, 509702144 bytes 2026-02-22T21:46:08.905 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1500 IOs, 566464512 bytes 2026-02-22T21:46:08.945 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1600 IOs, 601216000 bytes 2026-02-22T21:46:09.009 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1700 IOs, 674648064 bytes 2026-02-22T21:46:09.065 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1800 IOs, 753432576 bytes 2026-02-22T21:46:09.134 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1900 IOs, 820673536 bytes 2026-02-22T21:46:09.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2000 IOs, 885147648 bytes 2026-02-22T21:46:09.267 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2100 IOs, 962171904 bytes 2026-02-22T21:46:09.305 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2200 IOs, 985477120 bytes 2026-02-22T21:46:09.353 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2300 IOs, 1015425024 bytes 2026-02-22T21:46:09.397 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2400 IOs, 1052887040 bytes 2026-02-22T21:46:09.438 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2500 IOs, 1078240256 bytes 2026-02-22T21:46:09.488 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2600 IOs, 1113431040 bytes 2026-02-22T21:46:09.543 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2700 IOs, 1165146112 bytes 2026-02-22T21:46:09.589 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2800 IOs, 1202028544 bytes 2026-02-22T21:46:09.645 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2900 IOs, 1266134016 bytes 2026-02-22T21:46:09.704 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3000 IOs, 1325156352 bytes 2026-02-22T21:46:09.779 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3100 IOs, 1439206400 bytes 2026-02-22T21:46:09.812 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3200 IOs, 1440241664 bytes 2026-02-22T21:46:09.849 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3300 IOs, 1459281920 bytes 2026-02-22T21:46:09.891 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3400 IOs, 1482948608 bytes 2026-02-22T21:46:09.929 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3500 IOs, 1509828608 bytes 2026-02-22T21:46:09.978 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3600 IOs, 1552607232 bytes 2026-02-22T21:46:10.021 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3700 IOs, 1586867200 bytes 2026-02-22T21:46:10.072 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3800 IOs, 1628473344 bytes 2026-02-22T21:46:10.121 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3900 IOs, 1676864512 bytes 2026-02-22T21:46:10.159 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4000 IOs, 1696546816 bytes 2026-02-22T21:46:10.188 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4100 IOs, 1697138688 bytes 2026-02-22T21:46:10.239 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4200 IOs, 1740747776 bytes 2026-02-22T21:46:10.280 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4300 IOs, 1780070400 bytes 2026-02-22T21:46:10.353 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4400 IOs, 1863806976 bytes 2026-02-22T21:46:10.410 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4500 IOs, 1931017216 bytes 2026-02-22T21:46:10.442 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4600 IOs, 1931630592 bytes 2026-02-22T21:46:10.482 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4700 IOs, 1949069312 bytes 2026-02-22T21:46:10.511 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4800 IOs, 1949677568 bytes 2026-02-22T21:46:10.562 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4900 IOs, 1997046784 bytes 2026-02-22T21:46:10.617 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5000 IOs, 2050656256 bytes 2026-02-22T21:46:10.666 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5100 IOs, 2096675840 bytes 2026-02-22T21:46:10.711 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5200 IOs, 2143870976 bytes 2026-02-22T21:46:10.759 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5300 IOs, 2179532800 bytes 2026-02-22T21:46:10.810 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5400 IOs, 2256935936 bytes 2026-02-22T21:46:10.882 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5500 IOs, 2343204864 bytes 2026-02-22T21:46:10.931 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5600 IOs, 2392389632 bytes 2026-02-22T21:46:10.980 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5700 IOs, 2427888640 bytes 2026-02-22T21:46:11.012 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5800 IOs, 2428487680 bytes 2026-02-22T21:46:11.074 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5900 IOs, 2507324416 bytes 2026-02-22T21:46:11.103 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6000 IOs, 2507971584 bytes 2026-02-22T21:46:11.152 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6100 IOs, 2549133312 bytes 2026-02-22T21:46:11.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6200 IOs, 2564001792 bytes 2026-02-22T21:46:11.231 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6300 IOs, 2590442496 bytes 2026-02-22T21:46:11.304 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6400 IOs, 2686561280 bytes 2026-02-22T21:46:11.354 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6500 IOs, 2731470848 bytes 2026-02-22T21:46:11.409 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6600 IOs, 2791555072 bytes 2026-02-22T21:46:11.440 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6700 IOs, 2793145344 bytes 2026-02-22T21:46:11.502 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6800 IOs, 2854797312 bytes 2026-02-22T21:46:11.560 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6900 IOs, 2920564736 bytes 2026-02-22T21:46:11.590 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7000 IOs, 2924186624 bytes 2026-02-22T21:46:11.636 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7100 IOs, 2978769920 bytes 2026-02-22T21:46:11.731 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7200 IOs, 3107194880 bytes 2026-02-22T21:46:11.772 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7300 IOs, 3117941760 bytes 2026-02-22T21:46:11.814 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7400 IOs, 3148562432 bytes 2026-02-22T21:46:11.866 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7500 IOs, 3194511360 bytes 2026-02-22T21:46:11.924 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7600 IOs, 3253433344 bytes 2026-02-22T21:46:11.963 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7700 IOs, 3280013312 bytes 2026-02-22T21:46:12.032 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7800 IOs, 3368261632 bytes 2026-02-22T21:46:12.082 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7900 IOs, 3419763712 bytes 2026-02-22T21:46:12.158 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8000 IOs, 3531138048 bytes 2026-02-22T21:46:12.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8100 IOs, 3597000704 bytes 2026-02-22T21:46:12.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8200 IOs, 3609544704 bytes 2026-02-22T21:46:12.293 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8300 IOs, 3616249856 bytes 2026-02-22T21:46:12.336 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8400 IOs, 3656869888 bytes 2026-02-22T21:46:12.382 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8500 IOs, 3698933760 bytes 2026-02-22T21:46:12.439 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8600 IOs, 3757130752 bytes 2026-02-22T21:46:12.514 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8700 IOs, 3850967040 bytes 2026-02-22T21:46:12.557 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8800 IOs, 3883174912 bytes 2026-02-22T21:46:12.603 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8900 IOs, 3919769600 bytes 2026-02-22T21:46:12.658 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9000 IOs, 3974519808 bytes 2026-02-22T21:46:12.697 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9100 IOs, 3996427264 bytes 2026-02-22T21:46:12.738 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9200 IOs, 4019954688 bytes 2026-02-22T21:46:12.777 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9300 IOs, 4027862016 bytes 2026-02-22T21:46:12.852 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9400 IOs, 4133563392 bytes 2026-02-22T21:46:13.082 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9500 IOs, 4138619904 bytes 2026-02-22T21:46:13.121 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9600 IOs, 4178841600 bytes 2026-02-22T21:46:13.171 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9700 IOs, 4212042752 bytes 2026-02-22T21:46:13.214 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9800 IOs, 4240216064 bytes 2026-02-22T21:46:13.290 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9900 IOs, 4344902656 bytes 2026-02-22T21:46:13.328 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10000 IOs, 4368649216 bytes 2026-02-22T21:46:13.387 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10100 IOs, 4434684928 bytes 2026-02-22T21:46:13.452 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10200 IOs, 4526351360 bytes 2026-02-22T21:46:13.505 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10300 IOs, 4587546624 bytes 2026-02-22T21:46:13.565 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10400 IOs, 4633657344 bytes 2026-02-22T21:46:13.622 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10500 IOs, 4697276416 bytes 2026-02-22T21:46:13.655 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10600 IOs, 4697897984 bytes 2026-02-22T21:46:13.686 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10700 IOs, 4704789504 bytes 2026-02-22T21:46:13.753 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10800 IOs, 4796348416 bytes 2026-02-22T21:46:13.814 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10900 IOs, 4856682496 bytes 2026-02-22T21:46:13.847 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11000 IOs, 4862580736 bytes 2026-02-22T21:46:13.891 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11100 IOs, 4890865664 bytes 2026-02-22T21:46:13.937 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11200 IOs, 4926668800 bytes 2026-02-22T21:46:13.972 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11300 IOs, 4938157056 bytes 2026-02-22T21:46:14.009 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11400 IOs, 4961509376 bytes 2026-02-22T21:46:14.064 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11500 IOs, 5015179264 bytes 2026-02-22T21:46:14.105 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11600 IOs, 5036929024 bytes 2026-02-22T21:46:14.166 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11700 IOs, 5112719360 bytes 2026-02-22T21:46:14.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11800 IOs, 5132401664 bytes 2026-02-22T21:46:14.247 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11900 IOs, 5166211072 bytes 2026-02-22T21:46:14.288 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12000 IOs, 5196212224 bytes 2026-02-22T21:46:14.322 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12100 IOs, 5196857344 bytes 2026-02-22T21:46:14.356 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12200 IOs, 5205627904 bytes 2026-02-22T21:46:14.402 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12300 IOs, 5240456192 bytes 2026-02-22T21:46:14.448 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12400 IOs, 5281605632 bytes 2026-02-22T21:46:14.517 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12500 IOs, 5368705024 bytes 2026-02-22T21:46:14.571 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12600 IOs, 5406674944 bytes 2026-02-22T21:46:14.623 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12700 IOs, 5452609536 bytes 2026-02-22T21:46:14.653 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12800 IOs, 5453208576 bytes 2026-02-22T21:46:14.710 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12900 IOs, 5514979328 bytes 2026-02-22T21:46:14.764 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13000 IOs, 5564246016 bytes 2026-02-22T21:46:14.799 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13100 IOs, 5575223296 bytes 2026-02-22T21:46:14.849 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13200 IOs, 5614871552 bytes 2026-02-22T21:46:14.892 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13300 IOs, 5648464896 bytes 2026-02-22T21:46:14.938 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13400 IOs, 5685168128 bytes 2026-02-22T21:46:14.969 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13500 IOs, 5687313408 bytes 2026-02-22T21:46:15.014 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13600 IOs, 5724385280 bytes 2026-02-22T21:46:15.051 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13700 IOs, 5746820096 bytes 2026-02-22T21:46:15.080 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13800 IOs, 5747430400 bytes 2026-02-22T21:46:15.120 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13900 IOs, 5772644352 bytes 2026-02-22T21:46:15.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14000 IOs, 5827450880 bytes 2026-02-22T21:46:15.235 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14100 IOs, 5895273472 bytes 2026-02-22T21:46:15.283 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14200 IOs, 5936963584 bytes 2026-02-22T21:46:15.336 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14300 IOs, 5972305920 bytes 2026-02-22T21:46:15.396 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14400 IOs, 6035892224 bytes 2026-02-22T21:46:15.442 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14500 IOs, 6088245248 bytes 2026-02-22T21:46:15.510 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14600 IOs, 6164106240 bytes 2026-02-22T21:46:15.562 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14700 IOs, 6194162688 bytes 2026-02-22T21:46:15.605 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14800 IOs, 6215367680 bytes 2026-02-22T21:46:15.659 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14900 IOs, 6265201664 bytes 2026-02-22T21:46:15.698 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15000 IOs, 6283807744 bytes 2026-02-22T21:46:15.765 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15100 IOs, 6359797760 bytes 2026-02-22T21:46:15.795 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15200 IOs, 6360413184 bytes 2026-02-22T21:46:15.856 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15300 IOs, 6437347328 bytes 2026-02-22T21:46:15.920 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15400 IOs, 6511597568 bytes 2026-02-22T21:46:15.977 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15500 IOs, 6573313024 bytes 2026-02-22T21:46:16.028 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15600 IOs, 6623601664 bytes 2026-02-22T21:46:16.064 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15700 IOs, 6640308224 bytes 2026-02-22T21:46:16.120 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15800 IOs, 6699530240 bytes 2026-02-22T21:46:16.173 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15900 IOs, 6753390592 bytes 2026-02-22T21:46:16.219 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16000 IOs, 6787786752 bytes 2026-02-22T21:46:16.266 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16100 IOs, 6829299712 bytes 2026-02-22T21:46:16.306 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16200 IOs, 6872119296 bytes 2026-02-22T21:46:16.355 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16300 IOs, 6894306304 bytes 2026-02-22T21:46:16.384 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16400 IOs, 6895690752 bytes 2026-02-22T21:46:16.420 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16500 IOs, 6915006464 bytes 2026-02-22T21:46:16.471 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16600 IOs, 6962817024 bytes 2026-02-22T21:46:16.503 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16700 IOs, 6964144128 bytes 2026-02-22T21:46:16.552 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16800 IOs, 7011391488 bytes 2026-02-22T21:46:16.602 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16900 IOs, 7035954176 bytes 2026-02-22T21:46:16.631 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17000 IOs, 7036559360 bytes 2026-02-22T21:46:16.682 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17100 IOs, 7080956928 bytes 2026-02-22T21:46:16.722 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17200 IOs, 7108511744 bytes 2026-02-22T21:46:16.763 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17300 IOs, 7127165952 bytes 2026-02-22T21:46:16.823 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17400 IOs, 7194462208 bytes 2026-02-22T21:46:16.872 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17500 IOs, 7234771968 bytes 2026-02-22T21:46:16.948 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17600 IOs, 7338435584 bytes 2026-02-22T21:46:16.992 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17700 IOs, 7376136192 bytes 2026-02-22T21:46:17.025 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17800 IOs, 7393903616 bytes 2026-02-22T21:46:17.078 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17900 IOs, 7430165504 bytes 2026-02-22T21:46:17.127 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18000 IOs, 7482093568 bytes 2026-02-22T21:46:17.185 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18100 IOs, 7553906688 bytes 2026-02-22T21:46:17.248 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18200 IOs, 7627437056 bytes 2026-02-22T21:46:17.285 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18300 IOs, 7646660608 bytes 2026-02-22T21:46:17.329 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18400 IOs, 7668963328 bytes 2026-02-22T21:46:17.377 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18500 IOs, 7701308416 bytes 2026-02-22T21:46:17.420 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18600 IOs, 7735088128 bytes 2026-02-22T21:46:17.462 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18700 IOs, 7768755200 bytes 2026-02-22T21:46:17.509 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18800 IOs, 7802088448 bytes 2026-02-22T21:46:17.590 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18900 IOs, 7913867264 bytes 2026-02-22T21:46:17.630 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19000 IOs, 7924965376 bytes 2026-02-22T21:46:17.680 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19100 IOs, 7976330240 bytes 2026-02-22T21:46:17.731 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19200 IOs, 8007399424 bytes 2026-02-22T21:46:17.774 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19300 IOs, 8025336832 bytes 2026-02-22T21:46:17.855 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19400 IOs, 8128155648 bytes 2026-02-22T21:46:17.908 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19500 IOs, 8177934336 bytes 2026-02-22T21:46:17.945 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19600 IOs, 8191850496 bytes 2026-02-22T21:46:17.992 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19700 IOs, 8226335744 bytes 2026-02-22T21:46:18.032 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19800 IOs, 8248402944 bytes 2026-02-22T21:46:18.077 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19900 IOs, 8281615360 bytes 2026-02-22T21:46:18.117 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20000 IOs, 8303463424 bytes 2026-02-22T21:46:18.166 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20100 IOs, 8349667328 bytes 2026-02-22T21:46:18.219 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20200 IOs, 8383972352 bytes 2026-02-22T21:46:18.259 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20300 IOs, 8404093952 bytes 2026-02-22T21:46:18.307 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20400 IOs, 8438774784 bytes 2026-02-22T21:46:18.345 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20500 IOs, 8458785792 bytes 2026-02-22T21:46:18.391 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20600 IOs, 8493582336 bytes 2026-02-22T21:46:18.448 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20700 IOs, 8546341888 bytes 2026-02-22T21:46:18.484 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20800 IOs, 8555022336 bytes 2026-02-22T21:46:18.521 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20900 IOs, 8564887552 bytes 2026-02-22T21:46:18.564 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21000 IOs, 8575975424 bytes 2026-02-22T21:46:18.632 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21100 IOs, 8655370240 bytes 2026-02-22T21:46:18.701 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21200 IOs, 8745622528 bytes 2026-02-22T21:46:18.747 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21300 IOs, 8774295552 bytes 2026-02-22T21:46:18.794 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21400 IOs, 8813114368 bytes 2026-02-22T21:46:18.837 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21500 IOs, 8853899264 bytes 2026-02-22T21:46:18.882 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21600 IOs, 8881874944 bytes 2026-02-22T21:46:18.927 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21700 IOs, 8922570752 bytes 2026-02-22T21:46:18.966 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21800 IOs, 8940556288 bytes 2026-02-22T21:46:19.021 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21900 IOs, 8998365184 bytes 2026-02-22T21:46:19.073 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22000 IOs, 9041441792 bytes 2026-02-22T21:46:19.115 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22100 IOs, 9100622848 bytes 2026-02-22T21:46:19.157 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22200 IOs, 9107841024 bytes 2026-02-22T21:46:19.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22300 IOs, 9117232128 bytes 2026-02-22T21:46:19.245 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22400 IOs, 9158129664 bytes 2026-02-22T21:46:19.291 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22500 IOs, 9197891584 bytes 2026-02-22T21:46:19.323 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22600 IOs, 9206209536 bytes 2026-02-22T21:46:19.356 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22700 IOs, 9212737536 bytes 2026-02-22T21:46:19.403 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22800 IOs, 9255028736 bytes 2026-02-22T21:46:19.445 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22900 IOs, 9284426752 bytes 2026-02-22T21:46:19.495 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23000 IOs, 9336087552 bytes 2026-02-22T21:46:19.566 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23100 IOs, 9435204608 bytes 2026-02-22T21:46:19.624 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23200 IOs, 9485449216 bytes 2026-02-22T21:46:19.667 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23300 IOs, 9514819584 bytes 2026-02-22T21:46:19.708 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23400 IOs, 9538032640 bytes 2026-02-22T21:46:19.768 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23500 IOs, 9600566272 bytes 2026-02-22T21:46:19.799 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23600 IOs, 9604998144 bytes 2026-02-22T21:46:19.854 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23700 IOs, 9664275456 bytes 2026-02-22T21:46:19.936 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23800 IOs, 9775488000 bytes 2026-02-22T21:46:19.990 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23900 IOs, 9822961664 bytes 2026-02-22T21:46:20.042 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24000 IOs, 9888058368 bytes 2026-02-22T21:46:20.114 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24100 IOs, 9995110400 bytes 2026-02-22T21:46:20.159 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24200 IOs, 10011339776 bytes 2026-02-22T21:46:20.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24300 IOs, 10039830528 bytes 2026-02-22T21:46:20.254 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24400 IOs, 10110242816 bytes 2026-02-22T21:46:20.306 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24500 IOs, 10140177408 bytes 2026-02-22T21:46:20.360 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24600 IOs, 10191208448 bytes 2026-02-22T21:46:20.408 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24700 IOs, 10233409536 bytes 2026-02-22T21:46:20.461 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24800 IOs, 10284103680 bytes 2026-02-22T21:46:20.496 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24900 IOs, 10290927616 bytes 2026-02-22T21:46:20.554 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25000 IOs, 10352319488 bytes 2026-02-22T21:46:20.628 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25100 IOs, 10444253184 bytes 2026-02-22T21:46:20.667 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25200 IOs, 10475881472 bytes 2026-02-22T21:46:20.723 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25300 IOs, 10525872128 bytes 2026-02-22T21:46:20.761 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25400 IOs, 10561984512 bytes 2026-02-22T21:46:20.805 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25500 IOs, 10567528448 bytes 2026-02-22T21:46:20.855 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25600 IOs, 10615629824 bytes 2026-02-22T21:46:20.909 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25700 IOs, 10674365440 bytes 2026-02-22T21:46:20.971 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25800 IOs, 10741517312 bytes 2026-02-22T21:46:21.021 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25900 IOs, 10789335040 bytes 2026-02-22T21:46:21.080 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ error_code=124 2026-02-22T21:46:21.080 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set -e 2026-02-22T21:46:21.080 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' 124 -eq 124 ']' 2026-02-22T21:46:21.080 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:46:21.080 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T21:46:21.080 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:46:21.080 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:46:21.080 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:46:21.080 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state_pattern=up+replaying 2026-02-22T21:46:21.080 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local description_pattern= 2026-02-22T21:46:21.080 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local service_pattern= 2026-02-22T21:46:21.080 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:46:21.080 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T21:46:22.082 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T21:46:22.082 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:46:22.082 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:46:22.082 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:46:22.082 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state_pattern=up+replaying 2026-02-22T21:46:22.082 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local description_pattern= 2026-02-22T21:46:22.082 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local service_pattern= 2026-02-22T21:46:22.082 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status 2026-02-22T21:46:22.083 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ CEPH_ARGS= 2026-02-22T21:46:22.083 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:46:22.116 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ status='test: 2026-02-22T21:46:22.116 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: global_id: b410963b-cae9-4337-a6ef-da65321c9374 2026-02-22T21:46:22.116 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: state: up+replaying 2026-02-22T21:46:22.117 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":536526848.0,"last_snapshot_bytes":536526848,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771796708,"remote_snapshot_timestamp":1771796708,"replay_state":"idle"} 2026-02-22T21:46:22.117 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: service: mirror.1 on trial026 2026-02-22T21:46:22.117 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: last_update: 2026-02-22 21:46:02 2026-02-22T21:46:22.117 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: peer_sites: 2026-02-22T21:46:22.117 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: name: cluster2 2026-02-22T21:46:22.118 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: state: up+stopped 2026-02-22T21:46:22.118 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: description: local image is primary 2026-02-22T21:46:22.118 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: last_update: 2026-02-22 21:46:03' 2026-02-22T21:46:22.118 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T21:46:22.119 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: state: up+replaying 2026-02-22T21:46:22.120 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep '^ description: .*' 2026-02-22T21:46:22.120 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":536526848.0,"last_snapshot_bytes":536526848,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771796708,"remote_snapshot_timestamp":1771796708,"replay_state":"idle"} 2026-02-22T21:46:22.121 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n '' ']' 2026-02-22T21:46:22.121 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ echo up+replaying 2026-02-22T21:46:22.122 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep '^up+' 2026-02-22T21:46:22.122 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:up+replaying 2026-02-22T21:46:22.123 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:46:22.123 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: service: mirror.1 on trial026 2026-02-22T21:46:22.126 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local last_update 2026-02-22T21:46:22.126 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:46:22.126 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ last_update='2026-02-22 21:46:02' 2026-02-22T21:46:22.126 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 21:46:02' 2026-02-22T21:46:22.126 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:46:22.127 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:46:22.127 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:46:22.127 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state_pattern=up+replaying 2026-02-22T21:46:22.127 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local 'prev_last_update=2026-02-22 21:46:02' 2026-02-22T21:46:22.127 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status 2026-02-22T21:46:22.127 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ CEPH_ARGS= 2026-02-22T21:46:22.128 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:46:22.155 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ status='OKOKOK142104261mirror.1trial02619.2.3-689-gf90f4eaetrueOK43584361mirror.2trial02619.2.3-689-gf90f4eaefalseOK45034506mirror.0trial02619.2.3-689-gf90f4eaefalseOK45384541mirror.3trial02619.2.3-689-gf90f4eaefalseOKtestb410963b-cae9-4337-a6ef-da65321c9374up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":536526848.0,"last_snapshot_bytes":536526848,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771796708,"remote_snapshot_timestamp":1771796708,"replay_state":"idle"}42104261mirror.1trial0262026-02-22 21:46:02cluster2cfa81703-817a-4074-adbc-5e6e200790bfup+stoppedlocal image is primary2026-02-22 21:46:03' 2026-02-22T21:46:22.155 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local last_update state 2026-02-22T21:46:22.155 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:46:22.156 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ last_update='2026-02-22 21:46:02' 2026-02-22T21:46:22.157 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:46:22.158 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ state=up+replaying 2026-02-22T21:46:22.158 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ echo up+replaying 2026-02-22T21:46:22.158 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep up+replaying 2026-02-22T21:46:22.159 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:up+replaying 2026-02-22T21:46:22.160 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:46:22.160 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:46:22.160 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ snap_name=snap10 2026-02-22T21:46:22.160 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_snap cluster2 mirror test snap10 2026-02-22T21:46:22.161 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:46:22.161 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:46:22.161 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:46:22.161 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap10 2026-02-22T21:46:22.161 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap10 --debug-rbd=20 --debug-journaler=20 2026-02-22T21:46:52.134 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T21:46:52.134 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:46:52.134 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:46:52.134 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:46:52.134 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T21:46:52.135 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:46:52.135 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:46:52.135 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:46:52.135 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=started 2026-02-22T21:46:52.135 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T21:46:52.135 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:46:52.135 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T21:46:53.136 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T21:46:53.136 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:46:53.136 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:46:53.136 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:46:53.136 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=started 2026-02-22T21:46:53.136 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_result 2026-02-22T21:46:53.136 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=stopped 2026-02-22T21:46:53.137 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:46:53.137 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_instance=cluster1 2026-02-22T21:46:53.137 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:46:53.137 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T21:46:53.137 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance=cluster1 2026-02-22T21:46:53.137 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local loop_instance 2026-02-22T21:46:53.137 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:46:53.137 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sleep 0 2026-02-22T21:46:53.137 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ grep -i state 2026-02-22T21:46:53.140 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:46:53.140 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ seq 0 3 2026-02-22T21:46:53.141 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:46:53.141 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:46:53.141 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:0 2026-02-22T21:46:53.141 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:46:53.141 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:46:53.141 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:46:53.141 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:0 2026-02-22T21:46:53.141 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:46:53.141 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:46:53.141 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:46:53.141 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=0 2026-02-22T21:46:53.141 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:46:53.142 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:46:53.142 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:46:53.142 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=0 2026-02-22T21:46:53.142 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T21:46:53.142 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:46:53.142 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:46:53.142 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:46:53.142 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:46:53.142 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:46:53.142 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:0 2026-02-22T21:46:53.142 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:46:53.143 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:46:53.143 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:46:53.143 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T21:46:53.143 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:46:53.143 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:46:53.143 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:46:53.143 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=0 2026-02-22T21:46:53.143 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=0 2026-02-22T21:46:53.143 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:46:53.151 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:46:53.152 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:46:53.152 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:46:53.152 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:46:53.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T21:46:53.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:46:53.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T21:46:53.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:46:53.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-22T21:46:53.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:1 2026-02-22T21:46:53.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:46:53.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:46:53.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T21:46:53.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:1 2026-02-22T21:46:53.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:46:53.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:46:53.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:46:53.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=1 2026-02-22T21:46:53.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T21:46:53.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:46:53.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:46:53.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=1 2026-02-22T21:46:53.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T21:46:53.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T21:46:53.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:1 2026-02-22T21:46:53.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:46:53.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:46:53.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T21:46:53.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:1 2026-02-22T21:46:53.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:46:53.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:46:53.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:46:53.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T21:46:53.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T21:46:53.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:46:53.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:46:53.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=1 2026-02-22T21:46:53.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=1 2026-02-22T21:46:53.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T21:46:53.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:46:53.209 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:46:53.209 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:46:53.209 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-22T21:46:53.254 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ return 0 2026-02-22T21:46:53.255 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:46:53.255 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:46:53.256 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep -i Replaying 2026-02-22T21:46:53.256 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: "state": "Replaying" 2026-02-22T21:46:53.257 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_state=started 2026-02-22T21:46:53.257 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test started = started 2026-02-22T21:46:53.257 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:46:53.257 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2026-02-22T21:46:53.257 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T21:46:53.258 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:46:53.258 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:46:53.258 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:46:53.258 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = journal ']' 2026-02-22T21:46:53.258 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T21:46:53.258 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-22T21:46:53.258 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:46:53.258 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:46:53.258 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:46:53.258 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-22T21:46:55.134 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Snapshot ID: 32 2026-02-22T21:46:55.143 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test 2026-02-22T21:46:55.143 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T21:46:55.143 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:46:55.143 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:46:55.144 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:46:55.144 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster2-mirror-test.status 2026-02-22T21:46:55.144 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster2-mirror-test.status 2026-02-22T21:46:55.144 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T21:46:55.147 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_log=/tmp/tmp.j44LZPowal/cluster2-mirror-test.status 2026-02-22T21:46:55.147 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster1-mirror-test.status 2026-02-22T21:46:55.147 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster1-mirror-test.status 2026-02-22T21:46:55.147 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T21:46:55.148 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_status_log=/tmp/tmp.j44LZPowal/cluster1-mirror-test.status 2026-02-22T21:46:55.148 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.j44LZPowal/cluster2-mirror-test.status 2026-02-22T21:46:55.148 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:46:55.148 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:46:55.148 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:46:55.149 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster2-mirror-test.status 2026-02-22T21:46:55.149 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-22T21:46:55.149 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:46:55.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T21:46:55.180 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snapshot_id=32 2026-02-22T21:46:55.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:46:55.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:46:55.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.2 2026-02-22T21:46:55.383 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.j44LZPowal/cluster1-mirror-test.status 2026-02-22T21:46:55.383 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:46:55.383 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:46:55.383 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:46:55.384 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-test.status 2026-02-22T21:46:55.384 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T21:46:55.384 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:46:55.412 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:46:55.413 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:46:55.414 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T21:46:55.414 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T21:46:55.414 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T21:46:55.415 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T21:46:55.415 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 32 = '' 2026-02-22T21:46:55.415 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:46:55.415 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.4 2026-02-22T21:46:55.815 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.j44LZPowal/cluster1-mirror-test.status 2026-02-22T21:46:55.815 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:46:55.815 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:46:55.815 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:46:55.815 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-test.status 2026-02-22T21:46:55.816 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T21:46:55.816 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:46:55.849 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:46:55.849 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:46:55.850 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T21:46:55.850 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T21:46:55.850 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T21:46:55.851 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T21:46:55.851 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 32 = '' 2026-02-22T21:46:55.851 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:46:55.851 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.8 2026-02-22T21:46:56.653 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.j44LZPowal/cluster1-mirror-test.status 2026-02-22T21:46:56.653 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:46:56.653 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:46:56.653 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:46:56.654 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-test.status 2026-02-22T21:46:56.654 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T21:46:56.654 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:46:56.701 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:46:56.702 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:46:56.703 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T21:46:56.703 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T21:46:56.704 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T21:46:56.704 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T21:46:56.704 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 32 = '' 2026-02-22T21:46:56.705 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:46:56.705 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1.6 2026-02-22T21:46:57.741 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.650586987815495) > weight (0.25) 2026-02-22T21:46:57.741 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.48067030321447646) > weight (0.25) 2026-02-22T21:46:57.742 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.47874281175368927) > weight (0.25) 2026-02-22T21:46:57.742 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.3 with skip (0.8130760842427092) > weight (0.25) 2026-02-22T21:46:57.742 INFO:tasks.rbd_mirror_thrash:waiting for 68.31930951176815 secs before thrashing 2026-02-22T21:46:58.306 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.j44LZPowal/cluster1-mirror-test.status 2026-02-22T21:46:58.306 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:46:58.306 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:46:58.306 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:46:58.306 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-test.status 2026-02-22T21:46:58.306 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T21:46:58.306 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:46:58.337 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:46:58.339 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id=32 2026-02-22T21:46:58.339 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 32 = 32 2026-02-22T21:46:58.339 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:46:58.339 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snap_present cluster1 mirror test snap10 2026-02-22T21:46:58.339 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:46:58.339 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:46:58.339 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:46:58.339 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap10 2026-02-22T21:46:58.339 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T21:46:58.339 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T21:46:58.339 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T21:46:59.340 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 info mirror/test@snap10 2026-02-22T21:46:59.370 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'test': 2026-02-22T21:46:59.370 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 512 MiB in 128 objects 2026-02-22T21:46:59.370 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T21:46:59.370 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 20 2026-02-22T21:46:59.370 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 10a25aa14179 2026-02-22T21:46:59.370 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.10a25aa14179 2026-02-22T21:46:59.370 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T21:46:59.370 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock, non-primary 2026-02-22T21:46:59.370 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T21:46:59.370 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T21:46:59.371 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:38:05 2026 2026-02-22T21:46:59.371 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:45:13 2026 2026-02-22T21:46:59.371 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:38:05 2026 2026-02-22T21:46:59.371 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: protected: False 2026-02-22T21:46:59.371 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T21:46:59.371 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T21:46:59.371 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: b410963b-cae9-4337-a6ef-da65321c9374 2026-02-22T21:46:59.371 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: false 2026-02-22T21:46:59.372 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:46:59.372 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n snap9-clean ']' 2026-02-22T21:46:59.372 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ compare_image_snaps mirror test snap9-clean 2026-02-22T21:46:59.372 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:46:59.372 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:46:59.372 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap9-clean 2026-02-22T21:46:59.372 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local ret=0 2026-02-22T21:46:59.372 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local rmt_export=/tmp/tmp.j44LZPowal/cluster2-mirror-test.export 2026-02-22T21:46:59.373 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local loc_export=/tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:46:59.373 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:46:59.373 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror export test@snap9-clean /tmp/tmp.j44LZPowal/cluster2-mirror-test.export 2026-02-22T21:46:59.718 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-22T21:46:59.728 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 -p mirror export test@snap9-clean /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:00.092 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-22T21:47:00.105 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ cmp /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:00.289 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:00.416 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:47:00.416 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ compare_image_snaps mirror test snap10 2026-02-22T21:47:00.417 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:47:00.417 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:47:00.417 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap10 2026-02-22T21:47:00.417 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local ret=0 2026-02-22T21:47:00.417 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local rmt_export=/tmp/tmp.j44LZPowal/cluster2-mirror-test.export 2026-02-22T21:47:00.417 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local loc_export=/tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:00.417 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:00.418 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror export test@snap10 /tmp/tmp.j44LZPowal/cluster2-mirror-test.export 2026-02-22T21:47:00.961 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-22T21:47:00.972 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 -p mirror export test@snap10 /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:01.354 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-22T21:47:01.365 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ cmp /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:01.549 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:01.676 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:47:01.676 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ clean_snap_name=snap10-clean 2026-02-22T21:47:01.676 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_snap cluster2 mirror test snap10-clean 2026-02-22T21:47:01.676 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:47:01.676 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:47:01.676 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:47:01.676 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap10-clean 2026-02-22T21:47:01.676 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap10-clean --debug-rbd=20 --debug-journaler=20 2026-02-22T21:47:02.140 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T21:47:02.140 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:47:02.141 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:47:02.141 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:47:02.141 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T21:47:02.141 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:47:02.141 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:47:02.141 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:47:02.141 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=started 2026-02-22T21:47:02.141 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T21:47:02.141 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:47:02.141 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T21:47:03.142 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T21:47:03.142 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:47:03.142 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:47:03.142 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:47:03.142 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=started 2026-02-22T21:47:03.143 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_result 2026-02-22T21:47:03.143 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=stopped 2026-02-22T21:47:03.143 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:47:03.143 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ grep -i state 2026-02-22T21:47:03.143 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_instance=cluster1 2026-02-22T21:47:03.143 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:47:03.143 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T21:47:03.143 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance=cluster1 2026-02-22T21:47:03.143 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local loop_instance 2026-02-22T21:47:03.143 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:47:03.143 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sleep 0 2026-02-22T21:47:03.146 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:47:03.146 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ seq 0 3 2026-02-22T21:47:03.147 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:47:03.147 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:47:03.147 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:0 2026-02-22T21:47:03.147 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:47:03.147 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:47:03.147 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:47:03.147 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:0 2026-02-22T21:47:03.147 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:47:03.147 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:47:03.147 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:47:03.147 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=0 2026-02-22T21:47:03.148 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:47:03.148 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:47:03.148 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:47:03.148 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=0 2026-02-22T21:47:03.148 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T21:47:03.148 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:47:03.148 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:47:03.148 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:47:03.148 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:47:03.148 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:47:03.148 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:0 2026-02-22T21:47:03.148 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:47:03.149 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:47:03.149 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:47:03.149 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T21:47:03.149 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:47:03.149 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:47:03.149 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:47:03.149 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=0 2026-02-22T21:47:03.149 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=0 2026-02-22T21:47:03.149 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:47:03.157 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:03.157 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:03.157 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:03.158 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:47:03.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T21:47:03.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:47:03.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T21:47:03.203 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:47:03.203 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-22T21:47:03.203 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:1 2026-02-22T21:47:03.203 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:47:03.203 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:47:03.203 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T21:47:03.203 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:1 2026-02-22T21:47:03.203 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:47:03.203 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:47:03.203 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:47:03.203 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=1 2026-02-22T21:47:03.203 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T21:47:03.203 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:47:03.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:47:03.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=1 2026-02-22T21:47:03.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T21:47:03.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T21:47:03.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:1 2026-02-22T21:47:03.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:47:03.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:47:03.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T21:47:03.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:1 2026-02-22T21:47:03.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:47:03.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:47:03.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:47:03.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T21:47:03.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T21:47:03.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:47:03.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:47:03.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=1 2026-02-22T21:47:03.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=1 2026-02-22T21:47:03.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T21:47:03.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:47:03.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:47:03.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:47:03.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-22T21:47:03.263 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ return 0 2026-02-22T21:47:03.263 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:47:03.264 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:47:03.264 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep -i Replaying 2026-02-22T21:47:03.265 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: "state": "Replaying" 2026-02-22T21:47:03.265 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_state=started 2026-02-22T21:47:03.265 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test started = started 2026-02-22T21:47:03.265 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:47:03.265 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2026-02-22T21:47:03.265 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T21:47:03.265 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:47:03.265 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:47:03.265 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:47:03.266 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = journal ']' 2026-02-22T21:47:03.266 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T21:47:03.266 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-22T21:47:03.266 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:47:03.266 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:47:03.266 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:47:03.266 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-22T21:47:05.144 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Snapshot ID: 34 2026-02-22T21:47:05.154 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test 2026-02-22T21:47:05.154 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T21:47:05.154 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:47:05.154 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:47:05.154 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:47:05.155 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster2-mirror-test.status 2026-02-22T21:47:05.155 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster2-mirror-test.status 2026-02-22T21:47:05.155 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T21:47:05.156 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_log=/tmp/tmp.j44LZPowal/cluster2-mirror-test.status 2026-02-22T21:47:05.156 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster1-mirror-test.status 2026-02-22T21:47:05.157 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster1-mirror-test.status 2026-02-22T21:47:05.157 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T21:47:05.158 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_status_log=/tmp/tmp.j44LZPowal/cluster1-mirror-test.status 2026-02-22T21:47:05.158 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.j44LZPowal/cluster2-mirror-test.status 2026-02-22T21:47:05.158 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:47:05.158 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:47:05.158 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:47:05.158 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster2-mirror-test.status 2026-02-22T21:47:05.158 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-22T21:47:05.159 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:47:05.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T21:47:05.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snapshot_id=34 2026-02-22T21:47:05.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:47:05.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:47:05.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.2 2026-02-22T21:47:05.395 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.j44LZPowal/cluster1-mirror-test.status 2026-02-22T21:47:05.395 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:47:05.395 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:47:05.395 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:47:05.395 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-test.status 2026-02-22T21:47:05.395 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T21:47:05.395 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:47:05.427 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:47:05.427 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:47:05.429 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T21:47:05.429 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T21:47:05.429 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T21:47:05.429 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T21:47:05.429 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 34 = '' 2026-02-22T21:47:05.429 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:47:05.430 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.4 2026-02-22T21:47:05.830 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.j44LZPowal/cluster1-mirror-test.status 2026-02-22T21:47:05.830 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:47:05.831 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:47:05.831 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:47:05.831 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-test.status 2026-02-22T21:47:05.831 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T21:47:05.831 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:47:05.863 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:47:05.865 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id=34 2026-02-22T21:47:05.865 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 34 = 34 2026-02-22T21:47:05.865 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:47:05.866 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snap_present cluster1 mirror test snap10-clean 2026-02-22T21:47:05.866 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:47:05.866 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:47:05.866 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:47:05.866 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap10-clean 2026-02-22T21:47:05.866 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T21:47:05.866 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T21:47:05.866 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T21:47:06.866 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 info mirror/test@snap10-clean 2026-02-22T21:47:06.899 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'test': 2026-02-22T21:47:06.900 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 512 MiB in 128 objects 2026-02-22T21:47:06.900 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T21:47:06.900 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 21 2026-02-22T21:47:06.900 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 10a25aa14179 2026-02-22T21:47:06.900 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.10a25aa14179 2026-02-22T21:47:06.901 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T21:47:06.901 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock, non-primary 2026-02-22T21:47:06.901 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T21:47:06.901 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T21:47:06.901 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:38:05 2026 2026-02-22T21:47:06.902 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:46:59 2026 2026-02-22T21:47:06.902 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:38:05 2026 2026-02-22T21:47:06.902 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: protected: False 2026-02-22T21:47:06.902 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T21:47:06.902 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T21:47:06.902 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: b410963b-cae9-4337-a6ef-da65321c9374 2026-02-22T21:47:06.903 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: false 2026-02-22T21:47:06.905 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:47:06.905 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ seq 1 10 2026-02-22T21:47:06.906 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 10` 2026-02-22T21:47:06.906 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ snap_name=snap1 2026-02-22T21:47:06.906 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ compare_image_snaps mirror test snap1 2026-02-22T21:47:06.906 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:47:06.906 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:47:06.906 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap1 2026-02-22T21:47:06.907 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local ret=0 2026-02-22T21:47:06.907 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local rmt_export=/tmp/tmp.j44LZPowal/cluster2-mirror-test.export 2026-02-22T21:47:06.907 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local loc_export=/tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:06.907 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:06.907 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror export test@snap1 /tmp/tmp.j44LZPowal/cluster2-mirror-test.export 2026-02-22T21:47:07.246 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-22T21:47:07.258 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 -p mirror export test@snap1 /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:07.656 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-22T21:47:07.670 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ cmp /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:07.854 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:07.980 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:47:07.980 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ snap_name=snap1-clean 2026-02-22T21:47:07.981 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ compare_image_snaps mirror test snap1-clean 2026-02-22T21:47:07.981 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:47:07.981 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:47:07.981 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap1-clean 2026-02-22T21:47:07.981 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local ret=0 2026-02-22T21:47:07.981 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local rmt_export=/tmp/tmp.j44LZPowal/cluster2-mirror-test.export 2026-02-22T21:47:07.981 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local loc_export=/tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:07.981 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:07.981 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror export test@snap1-clean /tmp/tmp.j44LZPowal/cluster2-mirror-test.export 2026-02-22T21:47:08.319 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-22T21:47:08.329 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 -p mirror export test@snap1-clean /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:08.726 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-22T21:47:08.738 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ cmp /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:08.925 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:09.053 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:47:09.054 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 10` 2026-02-22T21:47:09.054 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ snap_name=snap2 2026-02-22T21:47:09.054 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ compare_image_snaps mirror test snap2 2026-02-22T21:47:09.054 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:47:09.054 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:47:09.054 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap2 2026-02-22T21:47:09.054 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local ret=0 2026-02-22T21:47:09.054 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local rmt_export=/tmp/tmp.j44LZPowal/cluster2-mirror-test.export 2026-02-22T21:47:09.054 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local loc_export=/tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:09.054 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:09.055 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror export test@snap2 /tmp/tmp.j44LZPowal/cluster2-mirror-test.export 2026-02-22T21:47:09.390 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-22T21:47:09.401 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 -p mirror export test@snap2 /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:09.786 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-22T21:47:09.798 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ cmp /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:09.986 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:10.116 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:47:10.116 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ snap_name=snap2-clean 2026-02-22T21:47:10.116 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ compare_image_snaps mirror test snap2-clean 2026-02-22T21:47:10.117 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:47:10.117 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:47:10.117 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap2-clean 2026-02-22T21:47:10.117 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local ret=0 2026-02-22T21:47:10.117 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local rmt_export=/tmp/tmp.j44LZPowal/cluster2-mirror-test.export 2026-02-22T21:47:10.117 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local loc_export=/tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:10.117 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:10.117 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror export test@snap2-clean /tmp/tmp.j44LZPowal/cluster2-mirror-test.export 2026-02-22T21:47:10.444 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-22T21:47:10.453 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 -p mirror export test@snap2-clean /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:10.837 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-22T21:47:10.848 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ cmp /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:11.031 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:11.154 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:47:11.154 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 10` 2026-02-22T21:47:11.154 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ snap_name=snap3 2026-02-22T21:47:11.154 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ compare_image_snaps mirror test snap3 2026-02-22T21:47:11.154 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:47:11.155 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:47:11.155 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap3 2026-02-22T21:47:11.155 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local ret=0 2026-02-22T21:47:11.155 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local rmt_export=/tmp/tmp.j44LZPowal/cluster2-mirror-test.export 2026-02-22T21:47:11.155 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local loc_export=/tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:11.155 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:11.155 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror export test@snap3 /tmp/tmp.j44LZPowal/cluster2-mirror-test.export 2026-02-22T21:47:11.480 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-22T21:47:11.493 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 -p mirror export test@snap3 /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:11.901 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-22T21:47:11.911 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ cmp /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:12.099 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:12.225 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:47:12.225 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ snap_name=snap3-clean 2026-02-22T21:47:12.225 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ compare_image_snaps mirror test snap3-clean 2026-02-22T21:47:12.225 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:47:12.225 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:47:12.225 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap3-clean 2026-02-22T21:47:12.225 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local ret=0 2026-02-22T21:47:12.225 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local rmt_export=/tmp/tmp.j44LZPowal/cluster2-mirror-test.export 2026-02-22T21:47:12.226 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local loc_export=/tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:12.226 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:12.226 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror export test@snap3-clean /tmp/tmp.j44LZPowal/cluster2-mirror-test.export 2026-02-22T21:47:12.567 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-22T21:47:12.577 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 -p mirror export test@snap3-clean /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:12.963 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-22T21:47:12.976 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ cmp /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:13.163 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:13.288 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:47:13.288 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 10` 2026-02-22T21:47:13.288 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ snap_name=snap4 2026-02-22T21:47:13.288 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ compare_image_snaps mirror test snap4 2026-02-22T21:47:13.288 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:47:13.288 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:47:13.288 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap4 2026-02-22T21:47:13.288 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local ret=0 2026-02-22T21:47:13.289 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local rmt_export=/tmp/tmp.j44LZPowal/cluster2-mirror-test.export 2026-02-22T21:47:13.289 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local loc_export=/tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:13.289 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:13.289 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror export test@snap4 /tmp/tmp.j44LZPowal/cluster2-mirror-test.export 2026-02-22T21:47:13.628 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-22T21:47:13.639 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 -p mirror export test@snap4 /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:14.020 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-22T21:47:14.032 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ cmp /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:14.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:14.350 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:47:14.351 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ snap_name=snap4-clean 2026-02-22T21:47:14.351 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ compare_image_snaps mirror test snap4-clean 2026-02-22T21:47:14.351 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:47:14.351 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:47:14.351 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap4-clean 2026-02-22T21:47:14.351 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local ret=0 2026-02-22T21:47:14.351 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local rmt_export=/tmp/tmp.j44LZPowal/cluster2-mirror-test.export 2026-02-22T21:47:14.351 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local loc_export=/tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:14.352 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:14.352 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror export test@snap4-clean /tmp/tmp.j44LZPowal/cluster2-mirror-test.export 2026-02-22T21:47:14.695 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-22T21:47:14.705 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 -p mirror export test@snap4-clean /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:15.108 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-22T21:47:15.120 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ cmp /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:15.307 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:15.435 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:47:15.436 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 10` 2026-02-22T21:47:15.436 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ snap_name=snap5 2026-02-22T21:47:15.436 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ compare_image_snaps mirror test snap5 2026-02-22T21:47:15.436 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:47:15.436 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:47:15.436 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap5 2026-02-22T21:47:15.436 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local ret=0 2026-02-22T21:47:15.436 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local rmt_export=/tmp/tmp.j44LZPowal/cluster2-mirror-test.export 2026-02-22T21:47:15.437 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local loc_export=/tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:15.437 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:15.437 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror export test@snap5 /tmp/tmp.j44LZPowal/cluster2-mirror-test.export 2026-02-22T21:47:15.775 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-22T21:47:15.786 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 -p mirror export test@snap5 /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:16.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-22T21:47:16.195 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ cmp /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:16.384 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:16.515 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:47:16.515 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ snap_name=snap5-clean 2026-02-22T21:47:16.515 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ compare_image_snaps mirror test snap5-clean 2026-02-22T21:47:16.515 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:47:16.516 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:47:16.516 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap5-clean 2026-02-22T21:47:16.516 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local ret=0 2026-02-22T21:47:16.516 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local rmt_export=/tmp/tmp.j44LZPowal/cluster2-mirror-test.export 2026-02-22T21:47:16.516 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local loc_export=/tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:16.516 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:16.516 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror export test@snap5-clean /tmp/tmp.j44LZPowal/cluster2-mirror-test.export 2026-02-22T21:47:16.864 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-22T21:47:16.874 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 -p mirror export test@snap5-clean /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:17.266 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-22T21:47:17.275 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ cmp /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:17.462 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:17.592 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:47:17.592 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 10` 2026-02-22T21:47:17.592 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ snap_name=snap6 2026-02-22T21:47:17.592 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ compare_image_snaps mirror test snap6 2026-02-22T21:47:17.592 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:47:17.593 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:47:17.593 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap6 2026-02-22T21:47:17.593 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local ret=0 2026-02-22T21:47:17.593 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local rmt_export=/tmp/tmp.j44LZPowal/cluster2-mirror-test.export 2026-02-22T21:47:17.593 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local loc_export=/tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:17.593 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:17.593 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror export test@snap6 /tmp/tmp.j44LZPowal/cluster2-mirror-test.export 2026-02-22T21:47:17.941 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-22T21:47:17.951 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 -p mirror export test@snap6 /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:18.329 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-22T21:47:18.342 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ cmp /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:18.530 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:18.659 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:47:18.659 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ snap_name=snap6-clean 2026-02-22T21:47:18.659 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ compare_image_snaps mirror test snap6-clean 2026-02-22T21:47:18.659 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:47:18.659 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:47:18.659 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap6-clean 2026-02-22T21:47:18.660 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local ret=0 2026-02-22T21:47:18.660 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local rmt_export=/tmp/tmp.j44LZPowal/cluster2-mirror-test.export 2026-02-22T21:47:18.660 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local loc_export=/tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:18.660 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:18.660 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror export test@snap6-clean /tmp/tmp.j44LZPowal/cluster2-mirror-test.export 2026-02-22T21:47:19.002 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-22T21:47:19.013 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 -p mirror export test@snap6-clean /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:19.417 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-22T21:47:19.428 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ cmp /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:19.616 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:19.742 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:47:19.743 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 10` 2026-02-22T21:47:19.743 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ snap_name=snap7 2026-02-22T21:47:19.743 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ compare_image_snaps mirror test snap7 2026-02-22T21:47:19.743 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:47:19.743 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:47:19.743 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap7 2026-02-22T21:47:19.743 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local ret=0 2026-02-22T21:47:19.743 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local rmt_export=/tmp/tmp.j44LZPowal/cluster2-mirror-test.export 2026-02-22T21:47:19.743 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local loc_export=/tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:19.743 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:19.744 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror export test@snap7 /tmp/tmp.j44LZPowal/cluster2-mirror-test.export 2026-02-22T21:47:20.086 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-22T21:47:20.097 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 -p mirror export test@snap7 /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:20.427 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.827654901275488) > weight (0.25) 2026-02-22T21:47:20.427 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.1 2026-02-22T21:47:20.427 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Sent signal 15 2026-02-22T21:47:20.427 INFO:tasks.rbd_mirror_thrash:waiting for 5.7960894186236755 secs before reviving daemons 2026-02-22T21:47:20.428 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial026.stderr:2026-02-22T21:47:20.427+0000 7fafdd6b0640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 (PID: 47986) UID: 1000 2026-02-22T21:47:20.499 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-22T21:47:20.512 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ cmp /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:20.696 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:20.824 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:47:20.825 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ snap_name=snap7-clean 2026-02-22T21:47:20.825 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ compare_image_snaps mirror test snap7-clean 2026-02-22T21:47:20.825 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:47:20.825 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:47:20.825 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap7-clean 2026-02-22T21:47:20.825 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local ret=0 2026-02-22T21:47:20.825 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local rmt_export=/tmp/tmp.j44LZPowal/cluster2-mirror-test.export 2026-02-22T21:47:20.825 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local loc_export=/tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:20.825 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:20.825 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror export test@snap7-clean /tmp/tmp.j44LZPowal/cluster2-mirror-test.export 2026-02-22T21:47:21.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-22T21:47:21.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 -p mirror export test@snap7-clean /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:21.575 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-22T21:47:21.586 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ cmp /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:21.773 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:21.902 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:47:21.903 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 10` 2026-02-22T21:47:21.903 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ snap_name=snap8 2026-02-22T21:47:21.903 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ compare_image_snaps mirror test snap8 2026-02-22T21:47:21.903 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:47:21.903 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:47:21.903 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap8 2026-02-22T21:47:21.903 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local ret=0 2026-02-22T21:47:21.903 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local rmt_export=/tmp/tmp.j44LZPowal/cluster2-mirror-test.export 2026-02-22T21:47:21.903 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local loc_export=/tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:21.903 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:21.904 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror export test@snap8 /tmp/tmp.j44LZPowal/cluster2-mirror-test.export 2026-02-22T21:47:22.249 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-22T21:47:22.262 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 -p mirror export test@snap8 /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:22.671 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-22T21:47:22.683 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ cmp /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:22.871 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:22.998 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:47:22.999 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ snap_name=snap8-clean 2026-02-22T21:47:22.999 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ compare_image_snaps mirror test snap8-clean 2026-02-22T21:47:22.999 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:47:22.999 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:47:22.999 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap8-clean 2026-02-22T21:47:22.999 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local ret=0 2026-02-22T21:47:22.999 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local rmt_export=/tmp/tmp.j44LZPowal/cluster2-mirror-test.export 2026-02-22T21:47:22.999 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local loc_export=/tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:22.999 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:23.000 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror export test@snap8-clean /tmp/tmp.j44LZPowal/cluster2-mirror-test.export 2026-02-22T21:47:23.338 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-22T21:47:23.349 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 -p mirror export test@snap8-clean /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:23.739 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-22T21:47:23.750 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ cmp /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:23.937 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:24.070 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:47:24.070 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 10` 2026-02-22T21:47:24.070 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ snap_name=snap9 2026-02-22T21:47:24.070 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ compare_image_snaps mirror test snap9 2026-02-22T21:47:24.070 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:47:24.070 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:47:24.071 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap9 2026-02-22T21:47:24.071 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local ret=0 2026-02-22T21:47:24.071 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local rmt_export=/tmp/tmp.j44LZPowal/cluster2-mirror-test.export 2026-02-22T21:47:24.071 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local loc_export=/tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:24.071 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:24.071 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror export test@snap9 /tmp/tmp.j44LZPowal/cluster2-mirror-test.export 2026-02-22T21:47:24.423 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-22T21:47:24.431 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 -p mirror export test@snap9 /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:24.845 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-22T21:47:24.856 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ cmp /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:25.047 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:25.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:47:25.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ snap_name=snap9-clean 2026-02-22T21:47:25.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ compare_image_snaps mirror test snap9-clean 2026-02-22T21:47:25.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:47:25.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:47:25.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap9-clean 2026-02-22T21:47:25.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local ret=0 2026-02-22T21:47:25.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local rmt_export=/tmp/tmp.j44LZPowal/cluster2-mirror-test.export 2026-02-22T21:47:25.180 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local loc_export=/tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:25.180 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:25.180 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror export test@snap9-clean /tmp/tmp.j44LZPowal/cluster2-mirror-test.export 2026-02-22T21:47:25.718 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-22T21:47:25.727 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 -p mirror export test@snap9-clean /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:26.121 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-22T21:47:26.133 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ cmp /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:26.224 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.1 2026-02-22T21:47:26.225 INFO:teuthology.orchestra.run:waiting for 600 2026-02-22T21:47:26.225 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.1 2026-02-22T21:47:26.225 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Restarting daemon 2026-02-22T21:47:26.225 DEBUG:teuthology.orchestra.run.trial026:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 2026-02-22T21:47:26.228 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Started 2026-02-22T21:47:26.228 INFO:tasks.rbd_mirror_thrash:waiting for 63.204460466616304 secs before thrashing 2026-02-22T21:47:26.327 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:26.454 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:47:26.454 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 10` 2026-02-22T21:47:26.455 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ snap_name=snap10 2026-02-22T21:47:26.455 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ compare_image_snaps mirror test snap10 2026-02-22T21:47:26.455 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:47:26.455 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:47:26.455 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap10 2026-02-22T21:47:26.455 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local ret=0 2026-02-22T21:47:26.455 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local rmt_export=/tmp/tmp.j44LZPowal/cluster2-mirror-test.export 2026-02-22T21:47:26.455 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local loc_export=/tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:26.455 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:26.455 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror export test@snap10 /tmp/tmp.j44LZPowal/cluster2-mirror-test.export 2026-02-22T21:47:26.807 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-22T21:47:26.817 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 -p mirror export test@snap10 /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:27.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-22T21:47:27.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ cmp /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:27.405 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:27.527 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:47:27.527 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ snap_name=snap10-clean 2026-02-22T21:47:27.527 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ compare_image_snaps mirror test snap10-clean 2026-02-22T21:47:27.528 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:47:27.528 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:47:27.528 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap10-clean 2026-02-22T21:47:27.528 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local ret=0 2026-02-22T21:47:27.528 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local rmt_export=/tmp/tmp.j44LZPowal/cluster2-mirror-test.export 2026-02-22T21:47:27.528 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local loc_export=/tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:27.528 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:27.528 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror export test@snap10-clean /tmp/tmp.j44LZPowal/cluster2-mirror-test.export 2026-02-22T21:47:27.856 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-22T21:47:27.867 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 -p mirror export test@snap10-clean /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:28.259 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-22T21:47:28.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ cmp /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:28.456 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-test.export /tmp/tmp.j44LZPowal/cluster1-mirror-test.export 2026-02-22T21:47:28.586 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:47:28.587 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ seq 1 10 2026-02-22T21:47:28.588 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 10` 2026-02-22T21:47:28.588 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ snap_name=snap1 2026-02-22T21:47:28.588 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_snapshot cluster2 mirror test snap1 2026-02-22T21:47:28.588 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:47:28.588 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:47:28.589 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:47:28.589 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap=snap1 2026-02-22T21:47:28.589 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap1 2026-02-22T21:47:29.127 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing snap: 100% complete...done. 2026-02-22T21:47:29.132 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ snap_name=snap1-clean 2026-02-22T21:47:29.132 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_snapshot cluster2 mirror test snap1-clean 2026-02-22T21:47:29.133 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:47:29.133 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:47:29.133 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:47:29.133 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap=snap1-clean 2026-02-22T21:47:29.133 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap1-clean 2026-02-22T21:47:30.133 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing snap: 100% complete...done. 2026-02-22T21:47:30.142 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 10` 2026-02-22T21:47:30.142 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ snap_name=snap2 2026-02-22T21:47:30.142 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_snapshot cluster2 mirror test snap2 2026-02-22T21:47:30.142 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:47:30.142 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:47:30.142 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:47:30.142 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap=snap2 2026-02-22T21:47:30.143 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap2 2026-02-22T21:47:31.137 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing snap: 100% complete...done. 2026-02-22T21:47:31.145 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ snap_name=snap2-clean 2026-02-22T21:47:31.145 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_snapshot cluster2 mirror test snap2-clean 2026-02-22T21:47:31.145 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:47:31.145 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:47:31.145 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:47:31.145 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap=snap2-clean 2026-02-22T21:47:31.145 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap2-clean 2026-02-22T21:47:32.138 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing snap: 100% complete...done. 2026-02-22T21:47:32.150 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 10` 2026-02-22T21:47:32.150 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ snap_name=snap3 2026-02-22T21:47:32.150 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_snapshot cluster2 mirror test snap3 2026-02-22T21:47:32.150 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:47:32.150 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:47:32.150 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:47:32.150 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap=snap3 2026-02-22T21:47:32.150 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap3 2026-02-22T21:47:33.139 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing snap: 100% complete...done. 2026-02-22T21:47:33.145 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ snap_name=snap3-clean 2026-02-22T21:47:33.145 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_snapshot cluster2 mirror test snap3-clean 2026-02-22T21:47:33.145 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:47:33.145 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:47:33.145 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:47:33.145 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap=snap3-clean 2026-02-22T21:47:33.145 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap3-clean 2026-02-22T21:47:34.121 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing snap: 100% complete...done. 2026-02-22T21:47:34.130 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 10` 2026-02-22T21:47:34.130 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ snap_name=snap4 2026-02-22T21:47:34.130 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_snapshot cluster2 mirror test snap4 2026-02-22T21:47:34.130 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:47:34.130 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:47:34.130 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:47:34.130 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap=snap4 2026-02-22T21:47:34.130 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap4 2026-02-22T21:47:35.143 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing snap: 100% complete...done. 2026-02-22T21:47:35.149 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ snap_name=snap4-clean 2026-02-22T21:47:35.149 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_snapshot cluster2 mirror test snap4-clean 2026-02-22T21:47:35.149 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:47:35.149 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:47:35.149 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:47:35.149 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap=snap4-clean 2026-02-22T21:47:35.149 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap4-clean 2026-02-22T21:47:36.139 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing snap: 100% complete...done. 2026-02-22T21:47:36.148 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 10` 2026-02-22T21:47:36.148 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ snap_name=snap5 2026-02-22T21:47:36.148 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_snapshot cluster2 mirror test snap5 2026-02-22T21:47:36.148 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:47:36.148 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:47:36.149 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:47:36.149 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap=snap5 2026-02-22T21:47:36.149 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap5 2026-02-22T21:47:37.142 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing snap: 100% complete...done. 2026-02-22T21:47:37.149 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ snap_name=snap5-clean 2026-02-22T21:47:37.149 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_snapshot cluster2 mirror test snap5-clean 2026-02-22T21:47:37.149 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:47:37.149 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:47:37.149 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:47:37.149 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap=snap5-clean 2026-02-22T21:47:37.149 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap5-clean 2026-02-22T21:47:38.149 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing snap: 100% complete...done. 2026-02-22T21:47:38.154 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 10` 2026-02-22T21:47:38.155 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ snap_name=snap6 2026-02-22T21:47:38.155 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_snapshot cluster2 mirror test snap6 2026-02-22T21:47:38.155 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:47:38.155 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:47:38.155 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:47:38.155 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap=snap6 2026-02-22T21:47:38.155 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap6 2026-02-22T21:47:39.122 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing snap: 100% complete...done. 2026-02-22T21:47:39.129 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ snap_name=snap6-clean 2026-02-22T21:47:39.129 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_snapshot cluster2 mirror test snap6-clean 2026-02-22T21:47:39.130 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:47:39.130 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:47:39.130 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:47:39.130 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap=snap6-clean 2026-02-22T21:47:39.130 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap6-clean 2026-02-22T21:47:40.152 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing snap: 100% complete...done. 2026-02-22T21:47:40.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 10` 2026-02-22T21:47:40.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ snap_name=snap7 2026-02-22T21:47:40.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_snapshot cluster2 mirror test snap7 2026-02-22T21:47:40.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:47:40.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:47:40.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:47:40.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap=snap7 2026-02-22T21:47:40.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap7 2026-02-22T21:47:41.155 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing snap: 100% complete...done. 2026-02-22T21:47:41.162 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ snap_name=snap7-clean 2026-02-22T21:47:41.162 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_snapshot cluster2 mirror test snap7-clean 2026-02-22T21:47:41.162 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:47:41.162 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:47:41.162 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:47:41.162 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap=snap7-clean 2026-02-22T21:47:41.162 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap7-clean 2026-02-22T21:47:42.161 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing snap: 100% complete...done. 2026-02-22T21:47:42.170 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 10` 2026-02-22T21:47:42.171 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ snap_name=snap8 2026-02-22T21:47:42.171 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_snapshot cluster2 mirror test snap8 2026-02-22T21:47:42.171 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:47:42.171 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:47:42.171 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:47:42.171 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap=snap8 2026-02-22T21:47:42.171 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap8 2026-02-22T21:47:43.154 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing snap: 100% complete...done. 2026-02-22T21:47:43.161 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ snap_name=snap8-clean 2026-02-22T21:47:43.161 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_snapshot cluster2 mirror test snap8-clean 2026-02-22T21:47:43.161 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:47:43.161 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:47:43.161 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:47:43.161 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap=snap8-clean 2026-02-22T21:47:43.161 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap8-clean 2026-02-22T21:47:44.120 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing snap: 100% complete...done. 2026-02-22T21:47:44.129 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 10` 2026-02-22T21:47:44.130 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ snap_name=snap9 2026-02-22T21:47:44.130 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_snapshot cluster2 mirror test snap9 2026-02-22T21:47:44.130 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:47:44.130 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:47:44.130 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:47:44.130 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap=snap9 2026-02-22T21:47:44.130 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap9 2026-02-22T21:47:45.159 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing snap: 100% complete...done. 2026-02-22T21:47:45.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ snap_name=snap9-clean 2026-02-22T21:47:45.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_snapshot cluster2 mirror test snap9-clean 2026-02-22T21:47:45.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:47:45.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:47:45.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:47:45.166 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap=snap9-clean 2026-02-22T21:47:45.166 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap9-clean 2026-02-22T21:47:46.160 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing snap: 100% complete...done. 2026-02-22T21:47:46.169 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 10` 2026-02-22T21:47:46.169 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ snap_name=snap10 2026-02-22T21:47:46.169 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_snapshot cluster2 mirror test snap10 2026-02-22T21:47:46.169 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:47:46.169 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:47:46.169 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:47:46.169 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap=snap10 2026-02-22T21:47:46.169 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap10 2026-02-22T21:47:47.160 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing snap: 100% complete...done. 2026-02-22T21:47:47.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ snap_name=snap10-clean 2026-02-22T21:47:47.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_snapshot cluster2 mirror test snap10-clean 2026-02-22T21:47:47.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:47:47.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:47:47.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:47:47.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap=snap10-clean 2026-02-22T21:47:47.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap10-clean 2026-02-22T21:47:48.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing snap: 100% complete...done. 2026-02-22T21:47:48.169 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image_retry cluster2 mirror test 2026-02-22T21:47:48.170 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:47:48.170 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:47:48.170 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:47:48.170 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T21:47:48.170 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0 2026-02-22T21:47:48.171 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image cluster2 mirror test 2026-02-22T21:47:48.171 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:47:48.171 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:47:48.171 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:47:48.171 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 snap purge mirror/test 2026-02-22T21:47:48.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 rm mirror/test 2026-02-22T21:47:48.239 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial026.stderr:2026-02-22T21:47:48.237+0000 7f478c757640 -1 rbd::mirror::ImageMap: 0x5625a682bd40 handle_peer_ack_remove: failed to remove global_image_id=b410963b-cae9-4337-a6ef-da65321c9374 2026-02-22T21:47:49.218 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2026-02-22T21:47:49.220 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:47:49.220 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_present cluster1 mirror test deleted 2026-02-22T21:47:49.220 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:47:49.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:47:49.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:47:49.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=deleted 2026-02-22T21:47:49.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T21:47:49.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T21:47:49.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T21:47:49.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror test 2026-02-22T21:47:49.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T21:47:49.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T21:47:49.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=test 2026-02-22T21:47:49.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-22T21:47:49.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T21:47:49.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image_id=10a25aa14179 2026-02-22T21:47:49.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T21:47:49.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.1 2026-02-22T21:47:49.355 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_present cluster1 mirror test deleted 10a25aa14179 2026-02-22T21:47:49.355 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:47:49.355 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:47:49.355 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:47:49.355 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=deleted 2026-02-22T21:47:49.355 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id=10a25aa14179 2026-02-22T21:47:49.355 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=deleted 2026-02-22T21:47:49.355 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_image_id 2026-02-22T21:47:49.355 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror test 2026-02-22T21:47:49.356 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T21:47:49.356 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T21:47:49.356 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=test 2026-02-22T21:47:49.356 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-22T21:47:49.356 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T21:47:49.383 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_image_id=10a25aa14179 2026-02-22T21:47:49.383 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n 10a25aa14179 2026-02-22T21:47:49.384 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -z 10a25aa14179 -o 10a25aa14179 = 10a25aa14179 2026-02-22T21:47:49.384 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_state=present 2026-02-22T21:47:49.384 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test deleted = present 2026-02-22T21:47:49.384 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T21:47:49.384 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T21:47:50.385 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_present cluster1 mirror test deleted 10a25aa14179 2026-02-22T21:47:50.386 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:47:50.386 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:47:50.386 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:47:50.386 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=deleted 2026-02-22T21:47:50.386 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id=10a25aa14179 2026-02-22T21:47:50.386 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=deleted 2026-02-22T21:47:50.386 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_image_id 2026-02-22T21:47:50.386 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror test 2026-02-22T21:47:50.386 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T21:47:50.386 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T21:47:50.386 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=test 2026-02-22T21:47:50.387 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-22T21:47:50.387 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T21:47:50.415 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_image_id=10a25aa14179 2026-02-22T21:47:50.415 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n 10a25aa14179 2026-02-22T21:47:50.415 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -z 10a25aa14179 -o 10a25aa14179 = 10a25aa14179 2026-02-22T21:47:50.415 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_state=present 2026-02-22T21:47:50.415 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test deleted = present 2026-02-22T21:47:50.415 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T21:47:50.416 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 2 2026-02-22T21:47:52.418 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_present cluster1 mirror test deleted 10a25aa14179 2026-02-22T21:47:52.418 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:47:52.418 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:47:52.418 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:47:52.419 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=deleted 2026-02-22T21:47:52.419 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id=10a25aa14179 2026-02-22T21:47:52.419 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=deleted 2026-02-22T21:47:52.419 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_image_id 2026-02-22T21:47:52.419 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror test 2026-02-22T21:47:52.419 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T21:47:52.419 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T21:47:52.419 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=test 2026-02-22T21:47:52.420 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-22T21:47:52.420 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T21:47:52.448 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_image_id=10a25aa14179 2026-02-22T21:47:52.448 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n 10a25aa14179 2026-02-22T21:47:52.448 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -z 10a25aa14179 -o 10a25aa14179 = 10a25aa14179 2026-02-22T21:47:52.448 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_state=present 2026-02-22T21:47:52.448 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test deleted = present 2026-02-22T21:47:52.448 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T21:47:52.449 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 4 2026-02-22T21:47:56.449 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_present cluster1 mirror test deleted 10a25aa14179 2026-02-22T21:47:56.450 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:47:56.450 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:47:56.450 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=test 2026-02-22T21:47:56.450 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=deleted 2026-02-22T21:47:56.450 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id=10a25aa14179 2026-02-22T21:47:56.450 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=deleted 2026-02-22T21:47:56.450 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_image_id 2026-02-22T21:47:56.450 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror test 2026-02-22T21:47:56.450 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T21:47:56.450 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T21:47:56.451 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=test 2026-02-22T21:47:56.451 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-22T21:47:56.451 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T21:47:56.475 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image test: (2) No such file or directory 2026-02-22T21:47:56.477 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_image_id= 2026-02-22T21:47:56.477 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T21:47:56.477 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test deleted = deleted 2026-02-22T21:47:56.478 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:47:56.478 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ testlog 'TEST: create many images' 2026-02-22T21:47:56.478 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ tee -a /tmp/tmp.j44LZPowal/rbd-mirror.test.log 2026-02-22T21:47:56.478 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ date '+%F %T' 2026-02-22T21:47:56.479 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ echo 2026-02-22 21:47:56 TEST: create many images 2026-02-22T21:47:56.479 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:2026-02-22 21:47:56 TEST: create many images 2026-02-22T21:47:56.479 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ snap_name=snap 2026-02-22T21:47:56.479 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ seq 1 50 2026-02-22T21:47:56.481 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T21:47:56.481 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_1 2026-02-22T21:47:56.481 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image_and_enable_mirror cluster2 mirror image_1 snapshot 128M 2026-02-22T21:47:56.481 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:47:56.481 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:47:56.481 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:47:56.481 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:47:56.481 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_1 2026-02-22T21:47:56.482 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:47:56.482 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T21:47:56.482 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n snapshot ']' 2026-02-22T21:47:56.482 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:47:56.482 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image cluster2 mirror image_1 128M 2026-02-22T21:47:56.482 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:47:56.482 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:47:56.482 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:47:56.482 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:47:56.482 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_1 2026-02-22T21:47:56.482 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:47:56.482 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local size=128 2026-02-22T21:47:56.483 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n 128M ']' 2026-02-22T21:47:56.483 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ size=128M 2026-02-22T21:47:56.483 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:47:56.483 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_1 2026-02-22T21:47:56.511 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T21:47:56.512 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:47:56.512 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' image = image ']' 2026-02-22T21:47:56.512 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:47:56.512 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ enable_mirror cluster2 mirror image_1 snapshot 2026-02-22T21:47:56.512 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:47:56.512 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:47:56.512 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_1 2026-02-22T21:47:56.512 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T21:47:56.512 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_1 snapshot 2026-02-22T21:47:57.128 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Mirroring enabled 2026-02-22T21:47:57.133 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 info mirror/image_1 2026-02-22T21:47:57.157 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_1': 2026-02-22T21:47:57.157 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 128 MiB in 32 objects 2026-02-22T21:47:57.157 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T21:47:57.158 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 1 2026-02-22T21:47:57.158 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 126b6d0fb07c 2026-02-22T21:47:57.158 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.126b6d0fb07c 2026-02-22T21:47:57.158 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T21:47:57.158 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock 2026-02-22T21:47:57.158 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T21:47:57.158 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T21:47:57.158 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:47:56 2026 2026-02-22T21:47:57.158 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:47:56 2026 2026-02-22T21:47:57.158 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:47:56 2026 2026-02-22T21:47:57.159 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T21:47:57.159 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T21:47:57.159 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: 90c63745-c08b-46eb-b4e8-44a4f1e91721 2026-02-22T21:47:57.159 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: true 2026-02-22T21:47:57.160 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n '' ']' 2026-02-22T21:47:57.160 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ stress_write_image cluster2 mirror image_1 2026-02-22T21:47:57.160 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:47:57.160 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:47:57.160 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_1 2026-02-22T21:47:57.160 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-22T21:47:57.161 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local duration=11 2026-02-22T21:47:57.161 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set +e 2026-02-22T21:47:57.161 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ timeout 11s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_1 --debug-rbd=20 --debug-journaler=20 2026-02-22T21:47:57.224 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:100 IOs, 8553472 bytes 2026-02-22T21:47:57.283 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:200 IOs, 88313856 bytes 2026-02-22T21:47:57.332 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:300 IOs, 155849728 bytes 2026-02-22T21:47:57.374 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:400 IOs, 198157312 bytes 2026-02-22T21:47:57.400 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:500 IOs, 198769664 bytes 2026-02-22T21:47:57.423 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:600 IOs, 199351296 bytes 2026-02-22T21:47:57.473 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:700 IOs, 270766080 bytes 2026-02-22T21:47:57.503 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:800 IOs, 290025472 bytes 2026-02-22T21:47:57.524 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:900 IOs, 290616320 bytes 2026-02-22T21:47:57.557 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1000 IOs, 304821248 bytes 2026-02-22T21:47:57.595 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1100 IOs, 335949824 bytes 2026-02-22T21:47:57.638 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1200 IOs, 376961024 bytes 2026-02-22T21:47:57.676 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1300 IOs, 420562944 bytes 2026-02-22T21:47:57.725 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1400 IOs, 454539264 bytes 2026-02-22T21:47:57.769 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1500 IOs, 493180928 bytes 2026-02-22T21:47:57.833 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1600 IOs, 581897216 bytes 2026-02-22T21:47:57.886 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1700 IOs, 659273728 bytes 2026-02-22T21:47:57.933 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1800 IOs, 718220288 bytes 2026-02-22T21:47:57.979 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1900 IOs, 758549504 bytes 2026-02-22T21:47:58.030 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2000 IOs, 821854208 bytes 2026-02-22T21:47:58.061 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2100 IOs, 834099200 bytes 2026-02-22T21:47:58.099 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2200 IOs, 867977216 bytes 2026-02-22T21:47:58.122 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2300 IOs, 868587520 bytes 2026-02-22T21:47:58.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2400 IOs, 922494976 bytes 2026-02-22T21:47:58.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2500 IOs, 923114496 bytes 2026-02-22T21:47:58.215 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2600 IOs, 931238912 bytes 2026-02-22T21:47:58.243 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2700 IOs, 931870720 bytes 2026-02-22T21:47:58.280 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2800 IOs, 966946816 bytes 2026-02-22T21:47:58.319 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2900 IOs, 985255936 bytes 2026-02-22T21:47:58.354 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3000 IOs, 1002822656 bytes 2026-02-22T21:47:58.389 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3100 IOs, 1019742208 bytes 2026-02-22T21:47:58.428 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3200 IOs, 1052203008 bytes 2026-02-22T21:47:58.466 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3300 IOs, 1084256256 bytes 2026-02-22T21:47:58.523 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3400 IOs, 1186434048 bytes 2026-02-22T21:47:58.568 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3500 IOs, 1212607488 bytes 2026-02-22T21:47:58.608 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3600 IOs, 1251181568 bytes 2026-02-22T21:47:58.644 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3700 IOs, 1270777856 bytes 2026-02-22T21:47:58.692 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3800 IOs, 1337744384 bytes 2026-02-22T21:47:58.743 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3900 IOs, 1408722944 bytes 2026-02-22T21:47:58.777 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4000 IOs, 1424643072 bytes 2026-02-22T21:47:58.806 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4100 IOs, 1431478272 bytes 2026-02-22T21:47:58.846 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4200 IOs, 1468954624 bytes 2026-02-22T21:47:58.891 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4300 IOs, 1519151104 bytes 2026-02-22T21:47:58.924 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4400 IOs, 1546029056 bytes 2026-02-22T21:47:58.981 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4500 IOs, 1609822208 bytes 2026-02-22T21:47:59.017 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4600 IOs, 1638312960 bytes 2026-02-22T21:47:59.046 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4700 IOs, 1658005504 bytes 2026-02-22T21:47:59.083 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4800 IOs, 1682174976 bytes 2026-02-22T21:47:59.141 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4900 IOs, 1758481408 bytes 2026-02-22T21:47:59.185 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5000 IOs, 1802408960 bytes 2026-02-22T21:47:59.218 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5100 IOs, 1812755456 bytes 2026-02-22T21:47:59.459 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5200 IOs, 1858929664 bytes 2026-02-22T21:47:59.507 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5300 IOs, 1894338560 bytes 2026-02-22T21:47:59.546 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5400 IOs, 1922779136 bytes 2026-02-22T21:47:59.589 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5500 IOs, 1961921536 bytes 2026-02-22T21:47:59.639 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5600 IOs, 2029133824 bytes 2026-02-22T21:47:59.688 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5700 IOs, 2079662080 bytes 2026-02-22T21:47:59.715 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5800 IOs, 2084234240 bytes 2026-02-22T21:47:59.746 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5900 IOs, 2101030912 bytes 2026-02-22T21:47:59.782 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6000 IOs, 2143107072 bytes 2026-02-22T21:47:59.823 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6100 IOs, 2166137856 bytes 2026-02-22T21:47:59.862 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6200 IOs, 2189885440 bytes 2026-02-22T21:47:59.894 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6300 IOs, 2215059456 bytes 2026-02-22T21:47:59.961 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6400 IOs, 2308314112 bytes 2026-02-22T21:48:00.018 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6500 IOs, 2367145984 bytes 2026-02-22T21:48:00.053 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6600 IOs, 2396002304 bytes 2026-02-22T21:48:00.088 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6700 IOs, 2417819648 bytes 2026-02-22T21:48:00.134 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6800 IOs, 2463113216 bytes 2026-02-22T21:48:00.163 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6900 IOs, 2477868032 bytes 2026-02-22T21:48:00.209 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7000 IOs, 2539719680 bytes 2026-02-22T21:48:00.245 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7100 IOs, 2567429120 bytes 2026-02-22T21:48:00.290 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7200 IOs, 2624780288 bytes 2026-02-22T21:48:00.364 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7300 IOs, 2740748288 bytes 2026-02-22T21:48:00.409 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7400 IOs, 2788108288 bytes 2026-02-22T21:48:00.446 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7500 IOs, 2837857280 bytes 2026-02-22T21:48:00.486 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7600 IOs, 2855127040 bytes 2026-02-22T21:48:00.516 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7700 IOs, 2871436288 bytes 2026-02-22T21:48:00.561 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7800 IOs, 2915253248 bytes 2026-02-22T21:48:00.596 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7900 IOs, 2944611328 bytes 2026-02-22T21:48:00.626 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8000 IOs, 2945195008 bytes 2026-02-22T21:48:00.652 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8100 IOs, 2948177920 bytes 2026-02-22T21:48:00.687 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8200 IOs, 2973323264 bytes 2026-02-22T21:48:00.717 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8300 IOs, 2990002176 bytes 2026-02-22T21:48:00.771 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8400 IOs, 3054894080 bytes 2026-02-22T21:48:00.813 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8500 IOs, 3093151744 bytes 2026-02-22T21:48:00.867 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8600 IOs, 3149373440 bytes 2026-02-22T21:48:00.895 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8700 IOs, 3158265856 bytes 2026-02-22T21:48:00.948 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8800 IOs, 3240272896 bytes 2026-02-22T21:48:01.026 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8900 IOs, 3340937216 bytes 2026-02-22T21:48:01.075 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9000 IOs, 3386591232 bytes 2026-02-22T21:48:01.119 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9100 IOs, 3432093696 bytes 2026-02-22T21:48:01.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9200 IOs, 3486586880 bytes 2026-02-22T21:48:01.203 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9300 IOs, 3517171712 bytes 2026-02-22T21:48:01.244 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9400 IOs, 3554485248 bytes 2026-02-22T21:48:01.294 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9500 IOs, 3608802304 bytes 2026-02-22T21:48:01.335 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9600 IOs, 3651906560 bytes 2026-02-22T21:48:01.370 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9700 IOs, 3671257088 bytes 2026-02-22T21:48:01.404 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9800 IOs, 3701707776 bytes 2026-02-22T21:48:01.452 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9900 IOs, 3756512256 bytes 2026-02-22T21:48:01.500 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10000 IOs, 3801090048 bytes 2026-02-22T21:48:01.547 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10100 IOs, 3859152896 bytes 2026-02-22T21:48:01.581 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10200 IOs, 3896381440 bytes 2026-02-22T21:48:01.617 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10300 IOs, 3897003008 bytes 2026-02-22T21:48:01.645 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10400 IOs, 3903863808 bytes 2026-02-22T21:48:01.709 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10500 IOs, 3999238144 bytes 2026-02-22T21:48:01.756 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10600 IOs, 4042417152 bytes 2026-02-22T21:48:01.787 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10700 IOs, 4057367552 bytes 2026-02-22T21:48:01.830 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10800 IOs, 4089609216 bytes 2026-02-22T21:48:01.887 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10900 IOs, 4145286144 bytes 2026-02-22T21:48:01.920 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11000 IOs, 4161306624 bytes 2026-02-22T21:48:01.971 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11100 IOs, 4220579840 bytes 2026-02-22T21:48:02.013 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11200 IOs, 4256032768 bytes 2026-02-22T21:48:02.063 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11300 IOs, 4292069376 bytes 2026-02-22T21:48:02.125 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11400 IOs, 4375936000 bytes 2026-02-22T21:48:02.151 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11500 IOs, 4376532992 bytes 2026-02-22T21:48:02.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11600 IOs, 4448256000 bytes 2026-02-22T21:48:02.257 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11700 IOs, 4510563328 bytes 2026-02-22T21:48:02.307 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11800 IOs, 4566815744 bytes 2026-02-22T21:48:02.355 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11900 IOs, 4610486272 bytes 2026-02-22T21:48:02.411 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12000 IOs, 4694830080 bytes 2026-02-22T21:48:02.451 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12100 IOs, 4722814976 bytes 2026-02-22T21:48:02.490 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12200 IOs, 4750547968 bytes 2026-02-22T21:48:02.522 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12300 IOs, 4770589696 bytes 2026-02-22T21:48:02.557 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12400 IOs, 4795398144 bytes 2026-02-22T21:48:02.606 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12500 IOs, 4848953344 bytes 2026-02-22T21:48:02.661 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12600 IOs, 4920271872 bytes 2026-02-22T21:48:02.740 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12700 IOs, 5055228928 bytes 2026-02-22T21:48:02.784 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12800 IOs, 5098500096 bytes 2026-02-22T21:48:02.811 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12900 IOs, 5099572224 bytes 2026-02-22T21:48:02.853 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13000 IOs, 5130087424 bytes 2026-02-22T21:48:02.901 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13100 IOs, 5180564480 bytes 2026-02-22T21:48:02.951 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13200 IOs, 5248629760 bytes 2026-02-22T21:48:02.986 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13300 IOs, 5269336064 bytes 2026-02-22T21:48:03.046 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13400 IOs, 5343544320 bytes 2026-02-22T21:48:03.088 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13500 IOs, 5381465088 bytes 2026-02-22T21:48:03.127 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13600 IOs, 5420971008 bytes 2026-02-22T21:48:03.162 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13700 IOs, 5438015488 bytes 2026-02-22T21:48:03.225 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13800 IOs, 5530558464 bytes 2026-02-22T21:48:03.257 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13900 IOs, 5543259136 bytes 2026-02-22T21:48:03.308 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14000 IOs, 5600019456 bytes 2026-02-22T21:48:03.347 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14100 IOs, 5653090304 bytes 2026-02-22T21:48:03.385 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14200 IOs, 5666758656 bytes 2026-02-22T21:48:03.419 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14300 IOs, 5696183296 bytes 2026-02-22T21:48:03.468 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14400 IOs, 5741358080 bytes 2026-02-22T21:48:03.505 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14500 IOs, 5767507968 bytes 2026-02-22T21:48:03.540 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14600 IOs, 5788400640 bytes 2026-02-22T21:48:03.582 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14700 IOs, 5824334848 bytes 2026-02-22T21:48:03.624 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14800 IOs, 5857822720 bytes 2026-02-22T21:48:03.679 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14900 IOs, 5921653760 bytes 2026-02-22T21:48:03.711 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15000 IOs, 5935813632 bytes 2026-02-22T21:48:03.744 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15100 IOs, 5963057152 bytes 2026-02-22T21:48:03.798 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15200 IOs, 6023540736 bytes 2026-02-22T21:48:03.854 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15300 IOs, 6089835520 bytes 2026-02-22T21:48:03.898 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15400 IOs, 6123061248 bytes 2026-02-22T21:48:03.940 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15500 IOs, 6169318400 bytes 2026-02-22T21:48:03.991 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15600 IOs, 6217805824 bytes 2026-02-22T21:48:04.034 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15700 IOs, 6246542336 bytes 2026-02-22T21:48:04.067 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15800 IOs, 6265542656 bytes 2026-02-22T21:48:04.120 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15900 IOs, 6335085568 bytes 2026-02-22T21:48:04.150 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16000 IOs, 6359973888 bytes 2026-02-22T21:48:04.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16100 IOs, 6360580096 bytes 2026-02-22T21:48:04.244 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16200 IOs, 6460230656 bytes 2026-02-22T21:48:04.279 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16300 IOs, 6476816384 bytes 2026-02-22T21:48:04.310 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16400 IOs, 6483887104 bytes 2026-02-22T21:48:04.336 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16500 IOs, 6489291776 bytes 2026-02-22T21:48:04.388 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16600 IOs, 6558404608 bytes 2026-02-22T21:48:04.433 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16700 IOs, 6603551744 bytes 2026-02-22T21:48:04.483 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16800 IOs, 6664840192 bytes 2026-02-22T21:48:04.528 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16900 IOs, 6711415808 bytes 2026-02-22T21:48:04.590 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17000 IOs, 6809937920 bytes 2026-02-22T21:48:04.662 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17100 IOs, 6931784704 bytes 2026-02-22T21:48:04.699 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17200 IOs, 6952214528 bytes 2026-02-22T21:48:04.745 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17300 IOs, 7012160512 bytes 2026-02-22T21:48:04.780 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17400 IOs, 7030085632 bytes 2026-02-22T21:48:04.824 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17500 IOs, 7078712320 bytes 2026-02-22T21:48:04.873 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17600 IOs, 7122057216 bytes 2026-02-22T21:48:04.914 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17700 IOs, 7151592448 bytes 2026-02-22T21:48:04.958 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17800 IOs, 7195632640 bytes 2026-02-22T21:48:05.001 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17900 IOs, 7234758656 bytes 2026-02-22T21:48:05.039 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18000 IOs, 7244580864 bytes 2026-02-22T21:48:05.094 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18100 IOs, 7325265920 bytes 2026-02-22T21:48:05.128 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18200 IOs, 7337084928 bytes 2026-02-22T21:48:05.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18300 IOs, 7444505600 bytes 2026-02-22T21:48:05.238 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18400 IOs, 7471014912 bytes 2026-02-22T21:48:05.277 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18500 IOs, 7500877824 bytes 2026-02-22T21:48:05.311 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18600 IOs, 7517142016 bytes 2026-02-22T21:48:05.354 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18700 IOs, 7558412288 bytes 2026-02-22T21:48:05.413 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18800 IOs, 7644892160 bytes 2026-02-22T21:48:05.457 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18900 IOs, 7687928832 bytes 2026-02-22T21:48:05.486 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19000 IOs, 7694749696 bytes 2026-02-22T21:48:05.541 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19100 IOs, 7774040064 bytes 2026-02-22T21:48:05.574 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19200 IOs, 7778860032 bytes 2026-02-22T21:48:05.616 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19300 IOs, 7822306304 bytes 2026-02-22T21:48:05.668 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19400 IOs, 7876510720 bytes 2026-02-22T21:48:05.730 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19500 IOs, 7968779264 bytes 2026-02-22T21:48:05.765 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19600 IOs, 7986558976 bytes 2026-02-22T21:48:05.793 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19700 IOs, 7988597760 bytes 2026-02-22T21:48:05.836 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19800 IOs, 8032391168 bytes 2026-02-22T21:48:05.870 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19900 IOs, 8051716096 bytes 2026-02-22T21:48:05.911 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20000 IOs, 8077652992 bytes 2026-02-22T21:48:05.952 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20100 IOs, 8111047680 bytes 2026-02-22T21:48:06.002 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20200 IOs, 8153296896 bytes 2026-02-22T21:48:06.045 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20300 IOs, 8175788032 bytes 2026-02-22T21:48:06.062 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.586555216021827) > weight (0.25) 2026-02-22T21:48:06.062 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.1 2026-02-22T21:48:06.062 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Sent signal 15 2026-02-22T21:48:06.062 INFO:tasks.rbd_mirror_thrash:waiting for 2.073750184426805 secs before reviving daemons 2026-02-22T21:48:06.063 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial026.stderr:2026-02-22T21:48:06.062+0000 7fe29dc34640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 (PID: 48197) UID: 1000 2026-02-22T21:48:06.085 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20400 IOs, 8210300928 bytes 2026-02-22T21:48:06.124 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20500 IOs, 8225548288 bytes 2026-02-22T21:48:06.174 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20600 IOs, 8285043712 bytes 2026-02-22T21:48:06.224 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20700 IOs, 8341748736 bytes 2026-02-22T21:48:06.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20800 IOs, 8348307456 bytes 2026-02-22T21:48:06.299 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20900 IOs, 8399219712 bytes 2026-02-22T21:48:06.330 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21000 IOs, 8409562112 bytes 2026-02-22T21:48:06.354 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21100 IOs, 8410747904 bytes 2026-02-22T21:48:06.381 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21200 IOs, 8416308224 bytes 2026-02-22T21:48:06.423 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21300 IOs, 8464194560 bytes 2026-02-22T21:48:06.462 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21400 IOs, 8498681856 bytes 2026-02-22T21:48:06.508 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21500 IOs, 8537878528 bytes 2026-02-22T21:48:06.548 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21600 IOs, 8586116096 bytes 2026-02-22T21:48:06.589 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21700 IOs, 8621563904 bytes 2026-02-22T21:48:06.616 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21800 IOs, 8622177280 bytes 2026-02-22T21:48:06.679 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21900 IOs, 8708701184 bytes 2026-02-22T21:48:06.723 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22000 IOs, 8749821952 bytes 2026-02-22T21:48:06.775 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22100 IOs, 8811108352 bytes 2026-02-22T21:48:06.810 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22200 IOs, 8847622144 bytes 2026-02-22T21:48:06.884 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22300 IOs, 8930802688 bytes 2026-02-22T21:48:06.946 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22400 IOs, 8992749568 bytes 2026-02-22T21:48:06.977 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22500 IOs, 8993369088 bytes 2026-02-22T21:48:07.011 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22600 IOs, 9002821632 bytes 2026-02-22T21:48:07.060 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22700 IOs, 9032037376 bytes 2026-02-22T21:48:07.110 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22800 IOs, 9093904384 bytes 2026-02-22T21:48:07.173 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22900 IOs, 9179929600 bytes 2026-02-22T21:48:07.218 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23000 IOs, 9224207360 bytes 2026-02-22T21:48:07.251 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23100 IOs, 9249134592 bytes 2026-02-22T21:48:07.290 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23200 IOs, 9272636416 bytes 2026-02-22T21:48:07.321 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23300 IOs, 9287982080 bytes 2026-02-22T21:48:07.382 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23400 IOs, 9359744000 bytes 2026-02-22T21:48:07.422 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23500 IOs, 9389058048 bytes 2026-02-22T21:48:07.478 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23600 IOs, 9455503360 bytes 2026-02-22T21:48:07.536 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23700 IOs, 9528080384 bytes 2026-02-22T21:48:07.567 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23800 IOs, 9540767744 bytes 2026-02-22T21:48:07.622 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23900 IOs, 9616740352 bytes 2026-02-22T21:48:07.665 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24000 IOs, 9649756160 bytes 2026-02-22T21:48:07.710 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24100 IOs, 9693913088 bytes 2026-02-22T21:48:07.736 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24200 IOs, 9694528512 bytes 2026-02-22T21:48:07.788 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24300 IOs, 9764410368 bytes 2026-02-22T21:48:07.825 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24400 IOs, 9779070976 bytes 2026-02-22T21:48:07.862 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24500 IOs, 9809080320 bytes 2026-02-22T21:48:07.907 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24600 IOs, 9844094976 bytes 2026-02-22T21:48:07.955 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24700 IOs, 9873765376 bytes 2026-02-22T21:48:08.002 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24800 IOs, 9923446784 bytes 2026-02-22T21:48:08.046 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24900 IOs, 9974509568 bytes 2026-02-22T21:48:08.092 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25000 IOs, 10001325056 bytes 2026-02-22T21:48:08.137 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.1 2026-02-22T21:48:08.137 INFO:teuthology.orchestra.run:waiting for 600 2026-02-22T21:48:08.138 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.1 2026-02-22T21:48:08.138 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Restarting daemon 2026-02-22T21:48:08.138 DEBUG:teuthology.orchestra.run.trial026:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 2026-02-22T21:48:08.141 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Started 2026-02-22T21:48:08.141 INFO:tasks.rbd_mirror_thrash:waiting for 95.53977907324281 secs before thrashing 2026-02-22T21:48:08.145 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25100 IOs, 10060624896 bytes 2026-02-22T21:48:08.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ error_code=124 2026-02-22T21:48:08.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set -e 2026-02-22T21:48:08.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' 124 -eq 124 ']' 2026-02-22T21:48:08.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:48:08.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T21:48:08.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_2 2026-02-22T21:48:08.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image_and_enable_mirror cluster2 mirror image_2 snapshot 128M 2026-02-22T21:48:08.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:48:08.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:48:08.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:48:08.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:48:08.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_2 2026-02-22T21:48:08.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:48:08.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T21:48:08.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n snapshot ']' 2026-02-22T21:48:08.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:48:08.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image cluster2 mirror image_2 128M 2026-02-22T21:48:08.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:48:08.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:48:08.166 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:48:08.166 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:48:08.166 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_2 2026-02-22T21:48:08.166 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:48:08.166 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local size=128 2026-02-22T21:48:08.166 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n 128M ']' 2026-02-22T21:48:08.166 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ size=128M 2026-02-22T21:48:08.166 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:48:08.166 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_2 2026-02-22T21:48:08.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T21:48:08.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:48:08.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' image = image ']' 2026-02-22T21:48:08.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:48:08.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ enable_mirror cluster2 mirror image_2 snapshot 2026-02-22T21:48:08.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:48:08.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:48:08.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_2 2026-02-22T21:48:08.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T21:48:08.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_2 snapshot 2026-02-22T21:48:09.137 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Mirroring enabled 2026-02-22T21:48:09.142 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 info mirror/image_2 2026-02-22T21:48:09.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_2': 2026-02-22T21:48:09.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 128 MiB in 32 objects 2026-02-22T21:48:09.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T21:48:09.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 1 2026-02-22T21:48:09.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 1277c5499589 2026-02-22T21:48:09.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.1277c5499589 2026-02-22T21:48:09.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T21:48:09.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock 2026-02-22T21:48:09.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T21:48:09.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T21:48:09.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:48:08 2026 2026-02-22T21:48:09.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:48:08 2026 2026-02-22T21:48:09.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:48:08 2026 2026-02-22T21:48:09.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T21:48:09.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T21:48:09.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: 9e277158-7fce-48dd-8563-7bdcdfac29d0 2026-02-22T21:48:09.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: true 2026-02-22T21:48:09.169 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n '' ']' 2026-02-22T21:48:09.169 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ stress_write_image cluster2 mirror image_2 2026-02-22T21:48:09.169 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:48:09.169 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:48:09.169 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_2 2026-02-22T21:48:09.169 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-22T21:48:09.170 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local duration=6 2026-02-22T21:48:09.170 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set +e 2026-02-22T21:48:09.170 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ timeout 6s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_2 --debug-rbd=20 --debug-journaler=20 2026-02-22T21:48:09.241 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:100 IOs, 18536448 bytes 2026-02-22T21:48:09.302 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:200 IOs, 107465728 bytes 2026-02-22T21:48:09.338 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:300 IOs, 137665536 bytes 2026-02-22T21:48:09.382 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:400 IOs, 200308736 bytes 2026-02-22T21:48:09.444 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:500 IOs, 281115648 bytes 2026-02-22T21:48:09.493 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:600 IOs, 325331968 bytes 2026-02-22T21:48:09.556 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:700 IOs, 417481728 bytes 2026-02-22T21:48:09.594 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:800 IOs, 469402624 bytes 2026-02-22T21:48:09.655 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:900 IOs, 546150400 bytes 2026-02-22T21:48:09.712 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1000 IOs, 624843776 bytes 2026-02-22T21:48:09.755 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1100 IOs, 659677184 bytes 2026-02-22T21:48:09.799 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1200 IOs, 712339456 bytes 2026-02-22T21:48:09.840 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1300 IOs, 751403008 bytes 2026-02-22T21:48:09.886 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1400 IOs, 805891072 bytes 2026-02-22T21:48:09.939 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1500 IOs, 868210688 bytes 2026-02-22T21:48:09.984 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1600 IOs, 909353984 bytes 2026-02-22T21:48:10.035 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1700 IOs, 976082944 bytes 2026-02-22T21:48:10.068 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1800 IOs, 997619712 bytes 2026-02-22T21:48:10.119 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1900 IOs, 1042411520 bytes 2026-02-22T21:48:10.154 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2000 IOs, 1068831744 bytes 2026-02-22T21:48:10.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2100 IOs, 1113335808 bytes 2026-02-22T21:48:10.242 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2200 IOs, 1167041536 bytes 2026-02-22T21:48:10.269 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2300 IOs, 1171722240 bytes 2026-02-22T21:48:10.297 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2400 IOs, 1176999936 bytes 2026-02-22T21:48:10.325 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2500 IOs, 1190056960 bytes 2026-02-22T21:48:10.368 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2600 IOs, 1244461056 bytes 2026-02-22T21:48:10.410 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2700 IOs, 1260152832 bytes 2026-02-22T21:48:10.450 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2800 IOs, 1293834240 bytes 2026-02-22T21:48:10.491 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2900 IOs, 1330699264 bytes 2026-02-22T21:48:10.518 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3000 IOs, 1342313472 bytes 2026-02-22T21:48:10.564 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3100 IOs, 1393921024 bytes 2026-02-22T21:48:10.602 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3200 IOs, 1427153920 bytes 2026-02-22T21:48:10.656 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3300 IOs, 1489643520 bytes 2026-02-22T21:48:10.704 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3400 IOs, 1536622592 bytes 2026-02-22T21:48:10.745 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3500 IOs, 1571103744 bytes 2026-02-22T21:48:10.779 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3600 IOs, 1589230592 bytes 2026-02-22T21:48:10.817 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3700 IOs, 1632050176 bytes 2026-02-22T21:48:10.877 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3800 IOs, 1707534336 bytes 2026-02-22T21:48:10.918 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3900 IOs, 1746792448 bytes 2026-02-22T21:48:10.991 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4000 IOs, 1844036608 bytes 2026-02-22T21:48:11.025 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4100 IOs, 1866073088 bytes 2026-02-22T21:48:11.062 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4200 IOs, 1895367680 bytes 2026-02-22T21:48:11.114 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4300 IOs, 1950529536 bytes 2026-02-22T21:48:11.142 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4400 IOs, 1957337088 bytes 2026-02-22T21:48:11.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4500 IOs, 2017673216 bytes 2026-02-22T21:48:11.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4600 IOs, 2061640704 bytes 2026-02-22T21:48:11.267 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4700 IOs, 2082069504 bytes 2026-02-22T21:48:11.315 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4800 IOs, 2152649728 bytes 2026-02-22T21:48:11.346 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4900 IOs, 2153260032 bytes 2026-02-22T21:48:11.380 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5000 IOs, 2178336768 bytes 2026-02-22T21:48:11.416 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5100 IOs, 2205480960 bytes 2026-02-22T21:48:11.447 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5200 IOs, 2221004800 bytes 2026-02-22T21:48:11.499 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5300 IOs, 2281673728 bytes 2026-02-22T21:48:11.567 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5400 IOs, 2379725824 bytes 2026-02-22T21:48:11.596 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5500 IOs, 2387295232 bytes 2026-02-22T21:48:11.633 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5600 IOs, 2416111616 bytes 2026-02-22T21:48:11.661 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5700 IOs, 2433609728 bytes 2026-02-22T21:48:11.696 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5800 IOs, 2450984960 bytes 2026-02-22T21:48:11.730 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5900 IOs, 2472183808 bytes 2026-02-22T21:48:11.764 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6000 IOs, 2494794752 bytes 2026-02-22T21:48:11.808 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6100 IOs, 2537228288 bytes 2026-02-22T21:48:11.852 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6200 IOs, 2584231936 bytes 2026-02-22T21:48:11.889 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6300 IOs, 2612214784 bytes 2026-02-22T21:48:11.933 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6400 IOs, 2655532032 bytes 2026-02-22T21:48:11.973 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6500 IOs, 2687783936 bytes 2026-02-22T21:48:12.029 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6600 IOs, 2744539136 bytes 2026-02-22T21:48:12.084 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6700 IOs, 2802639872 bytes 2026-02-22T21:48:12.131 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6800 IOs, 2859392000 bytes 2026-02-22T21:48:12.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6900 IOs, 2913763328 bytes 2026-02-22T21:48:12.214 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7000 IOs, 2949137408 bytes 2026-02-22T21:48:12.262 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7100 IOs, 3005547520 bytes 2026-02-22T21:48:12.319 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7200 IOs, 3072657408 bytes 2026-02-22T21:48:12.353 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7300 IOs, 3096123392 bytes 2026-02-22T21:48:12.413 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7400 IOs, 3173970944 bytes 2026-02-22T21:48:12.459 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7500 IOs, 3233589248 bytes 2026-02-22T21:48:12.494 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7600 IOs, 3239762944 bytes 2026-02-22T21:48:12.536 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7700 IOs, 3273035776 bytes 2026-02-22T21:48:12.564 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7800 IOs, 3275095040 bytes 2026-02-22T21:48:12.597 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7900 IOs, 3286589440 bytes 2026-02-22T21:48:12.658 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8000 IOs, 3369355264 bytes 2026-02-22T21:48:12.696 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8100 IOs, 3388620800 bytes 2026-02-22T21:48:12.757 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8200 IOs, 3469791232 bytes 2026-02-22T21:48:12.798 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8300 IOs, 3500072960 bytes 2026-02-22T21:48:12.830 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8400 IOs, 3537393664 bytes 2026-02-22T21:48:12.914 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8500 IOs, 3668456448 bytes 2026-02-22T21:48:12.974 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8600 IOs, 3743511552 bytes 2026-02-22T21:48:13.015 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8700 IOs, 3770063872 bytes 2026-02-22T21:48:13.053 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8800 IOs, 3797547008 bytes 2026-02-22T21:48:13.107 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8900 IOs, 3858163712 bytes 2026-02-22T21:48:13.152 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9000 IOs, 3897401344 bytes 2026-02-22T21:48:13.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9100 IOs, 3941382144 bytes 2026-02-22T21:48:13.248 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9200 IOs, 4004229120 bytes 2026-02-22T21:48:13.290 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9300 IOs, 4033967104 bytes 2026-02-22T21:48:13.329 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9400 IOs, 4063909888 bytes 2026-02-22T21:48:13.365 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9500 IOs, 4084636672 bytes 2026-02-22T21:48:13.400 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9600 IOs, 4111868928 bytes 2026-02-22T21:48:13.443 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9700 IOs, 4138752000 bytes 2026-02-22T21:48:13.481 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9800 IOs, 4169214976 bytes 2026-02-22T21:48:13.524 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9900 IOs, 4205096960 bytes 2026-02-22T21:48:13.563 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10000 IOs, 4241695744 bytes 2026-02-22T21:48:13.610 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10100 IOs, 4299950080 bytes 2026-02-22T21:48:13.652 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10200 IOs, 4340699136 bytes 2026-02-22T21:48:13.679 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10300 IOs, 4343876608 bytes 2026-02-22T21:48:13.710 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10400 IOs, 4357720064 bytes 2026-02-22T21:48:13.752 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10500 IOs, 4390118400 bytes 2026-02-22T21:48:13.784 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10600 IOs, 4405875712 bytes 2026-02-22T21:48:13.832 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10700 IOs, 4451614720 bytes 2026-02-22T21:48:13.883 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10800 IOs, 4503209984 bytes 2026-02-22T21:48:13.922 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10900 IOs, 4517689344 bytes 2026-02-22T21:48:13.960 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11000 IOs, 4561476608 bytes 2026-02-22T21:48:14.016 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11100 IOs, 4640751616 bytes 2026-02-22T21:48:14.071 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11200 IOs, 4688496640 bytes 2026-02-22T21:48:14.118 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11300 IOs, 4721572864 bytes 2026-02-22T21:48:14.146 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11400 IOs, 4726349824 bytes 2026-02-22T21:48:14.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11500 IOs, 4766970880 bytes 2026-02-22T21:48:14.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11600 IOs, 4767581184 bytes 2026-02-22T21:48:14.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11700 IOs, 4867418112 bytes 2026-02-22T21:48:14.329 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11800 IOs, 4905134080 bytes 2026-02-22T21:48:14.373 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11900 IOs, 4948618240 bytes 2026-02-22T21:48:14.412 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12000 IOs, 4977046528 bytes 2026-02-22T21:48:14.442 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12100 IOs, 4986858496 bytes 2026-02-22T21:48:14.507 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12200 IOs, 5092742144 bytes 2026-02-22T21:48:14.567 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12300 IOs, 5178039296 bytes 2026-02-22T21:48:14.607 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12400 IOs, 5220425728 bytes 2026-02-22T21:48:14.668 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12500 IOs, 5312433152 bytes 2026-02-22T21:48:14.732 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12600 IOs, 5395027968 bytes 2026-02-22T21:48:14.788 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12700 IOs, 5471493120 bytes 2026-02-22T21:48:14.818 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12800 IOs, 5478063104 bytes 2026-02-22T21:48:14.853 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12900 IOs, 5516062720 bytes 2026-02-22T21:48:14.902 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13000 IOs, 5565296640 bytes 2026-02-22T21:48:14.941 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13100 IOs, 5583214592 bytes 2026-02-22T21:48:14.994 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13200 IOs, 5638801408 bytes 2026-02-22T21:48:15.051 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13300 IOs, 5718853632 bytes 2026-02-22T21:48:15.101 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13400 IOs, 5769275392 bytes 2026-02-22T21:48:15.154 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13500 IOs, 5825181696 bytes 2026-02-22T21:48:15.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ error_code=124 2026-02-22T21:48:15.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set -e 2026-02-22T21:48:15.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' 124 -eq 124 ']' 2026-02-22T21:48:15.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:48:15.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T21:48:15.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_3 2026-02-22T21:48:15.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image_and_enable_mirror cluster2 mirror image_3 snapshot 128M 2026-02-22T21:48:15.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:48:15.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:48:15.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:48:15.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:48:15.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_3 2026-02-22T21:48:15.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:48:15.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T21:48:15.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n snapshot ']' 2026-02-22T21:48:15.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:48:15.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image cluster2 mirror image_3 128M 2026-02-22T21:48:15.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:48:15.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:48:15.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:48:15.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:48:15.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_3 2026-02-22T21:48:15.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:48:15.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local size=128 2026-02-22T21:48:15.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n 128M ']' 2026-02-22T21:48:15.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ size=128M 2026-02-22T21:48:15.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:48:15.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_3 2026-02-22T21:48:15.203 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T21:48:15.203 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:48:15.203 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' image = image ']' 2026-02-22T21:48:15.203 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:48:15.203 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ enable_mirror cluster2 mirror image_3 snapshot 2026-02-22T21:48:15.203 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:48:15.203 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:48:15.203 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_3 2026-02-22T21:48:15.203 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T21:48:15.203 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_3 snapshot 2026-02-22T21:48:16.139 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Mirroring enabled 2026-02-22T21:48:16.145 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 info mirror/image_3 2026-02-22T21:48:16.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_3': 2026-02-22T21:48:16.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 128 MiB in 32 objects 2026-02-22T21:48:16.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T21:48:16.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 1 2026-02-22T21:48:16.169 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 128a34ee05cb 2026-02-22T21:48:16.169 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.128a34ee05cb 2026-02-22T21:48:16.169 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T21:48:16.169 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock 2026-02-22T21:48:16.169 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T21:48:16.169 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T21:48:16.169 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:48:15 2026 2026-02-22T21:48:16.169 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:48:15 2026 2026-02-22T21:48:16.169 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:48:15 2026 2026-02-22T21:48:16.169 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T21:48:16.169 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T21:48:16.170 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: 06da309f-d597-43fe-8ebc-0e374ae46a50 2026-02-22T21:48:16.170 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: true 2026-02-22T21:48:16.170 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n '' ']' 2026-02-22T21:48:16.171 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ stress_write_image cluster2 mirror image_3 2026-02-22T21:48:16.171 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:48:16.171 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:48:16.171 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_3 2026-02-22T21:48:16.171 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-22T21:48:16.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local duration=5 2026-02-22T21:48:16.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set +e 2026-02-22T21:48:16.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ timeout 5s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_3 --debug-rbd=20 --debug-journaler=20 2026-02-22T21:48:16.241 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:100 IOs, 23218176 bytes 2026-02-22T21:48:16.289 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:200 IOs, 71147520 bytes 2026-02-22T21:48:16.336 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:300 IOs, 123276288 bytes 2026-02-22T21:48:16.370 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:400 IOs, 135920640 bytes 2026-02-22T21:48:16.419 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:500 IOs, 181270528 bytes 2026-02-22T21:48:16.470 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:600 IOs, 248040448 bytes 2026-02-22T21:48:16.503 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:700 IOs, 251663360 bytes 2026-02-22T21:48:16.550 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:800 IOs, 297436160 bytes 2026-02-22T21:48:16.585 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:900 IOs, 316723200 bytes 2026-02-22T21:48:16.618 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1000 IOs, 337664000 bytes 2026-02-22T21:48:16.672 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1100 IOs, 401577984 bytes 2026-02-22T21:48:16.718 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1200 IOs, 460878848 bytes 2026-02-22T21:48:16.762 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1300 IOs, 484934656 bytes 2026-02-22T21:48:16.789 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1400 IOs, 485555200 bytes 2026-02-22T21:48:16.822 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1500 IOs, 512262144 bytes 2026-02-22T21:48:16.884 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1600 IOs, 582097920 bytes 2026-02-22T21:48:16.927 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1700 IOs, 615870464 bytes 2026-02-22T21:48:16.971 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1800 IOs, 681289728 bytes 2026-02-22T21:48:17.020 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1900 IOs, 696944640 bytes 2026-02-22T21:48:17.066 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2000 IOs, 753782784 bytes 2026-02-22T21:48:17.114 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2100 IOs, 788898816 bytes 2026-02-22T21:48:17.151 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2200 IOs, 809170944 bytes 2026-02-22T21:48:17.215 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2300 IOs, 886730752 bytes 2026-02-22T21:48:17.270 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2400 IOs, 952372224 bytes 2026-02-22T21:48:17.324 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2500 IOs, 1024314368 bytes 2026-02-22T21:48:17.360 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2600 IOs, 1048360960 bytes 2026-02-22T21:48:17.411 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2700 IOs, 1107324928 bytes 2026-02-22T21:48:17.446 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2800 IOs, 1125372928 bytes 2026-02-22T21:48:17.485 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2900 IOs, 1172926464 bytes 2026-02-22T21:48:17.546 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3000 IOs, 1237118976 bytes 2026-02-22T21:48:17.577 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3100 IOs, 1242016768 bytes 2026-02-22T21:48:17.616 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3200 IOs, 1272728576 bytes 2026-02-22T21:48:17.655 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3300 IOs, 1307586560 bytes 2026-02-22T21:48:17.712 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3400 IOs, 1385661440 bytes 2026-02-22T21:48:17.747 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3500 IOs, 1395316736 bytes 2026-02-22T21:48:17.773 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3600 IOs, 1395942400 bytes 2026-02-22T21:48:17.806 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3700 IOs, 1420509184 bytes 2026-02-22T21:48:17.856 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3800 IOs, 1456401408 bytes 2026-02-22T21:48:17.905 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3900 IOs, 1509054464 bytes 2026-02-22T21:48:17.942 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4000 IOs, 1522423808 bytes 2026-02-22T21:48:17.977 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4100 IOs, 1535815680 bytes 2026-02-22T21:48:18.018 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4200 IOs, 1545281536 bytes 2026-02-22T21:48:18.071 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4300 IOs, 1604103168 bytes 2026-02-22T21:48:18.121 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4400 IOs, 1653493760 bytes 2026-02-22T21:48:18.169 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4500 IOs, 1692022784 bytes 2026-02-22T21:48:18.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4600 IOs, 1695814656 bytes 2026-02-22T21:48:18.251 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4700 IOs, 1747136512 bytes 2026-02-22T21:48:18.286 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4800 IOs, 1768431616 bytes 2026-02-22T21:48:18.355 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4900 IOs, 1856492544 bytes 2026-02-22T21:48:18.426 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5000 IOs, 1958316032 bytes 2026-02-22T21:48:18.453 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5100 IOs, 1958943744 bytes 2026-02-22T21:48:18.489 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5200 IOs, 1982843904 bytes 2026-02-22T21:48:18.530 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5300 IOs, 2023624704 bytes 2026-02-22T21:48:18.566 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5400 IOs, 2043444224 bytes 2026-02-22T21:48:18.608 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5500 IOs, 2093155328 bytes 2026-02-22T21:48:18.654 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5600 IOs, 2128464896 bytes 2026-02-22T21:48:18.712 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5700 IOs, 2205579264 bytes 2026-02-22T21:48:18.754 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5800 IOs, 2235365376 bytes 2026-02-22T21:48:18.794 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5900 IOs, 2263324672 bytes 2026-02-22T21:48:18.840 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6000 IOs, 2300502016 bytes 2026-02-22T21:48:18.876 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6100 IOs, 2323885056 bytes 2026-02-22T21:48:18.948 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6200 IOs, 2436218880 bytes 2026-02-22T21:48:18.991 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6300 IOs, 2484209664 bytes 2026-02-22T21:48:19.048 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6400 IOs, 2530880512 bytes 2026-02-22T21:48:19.109 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6500 IOs, 2615459840 bytes 2026-02-22T21:48:19.163 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6600 IOs, 2644386816 bytes 2026-02-22T21:48:19.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6700 IOs, 2675227648 bytes 2026-02-22T21:48:19.258 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6800 IOs, 2740490240 bytes 2026-02-22T21:48:19.291 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6900 IOs, 2741087232 bytes 2026-02-22T21:48:19.342 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7000 IOs, 2796749824 bytes 2026-02-22T21:48:19.397 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7100 IOs, 2870697984 bytes 2026-02-22T21:48:19.459 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7200 IOs, 2948660224 bytes 2026-02-22T21:48:19.534 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7300 IOs, 3078364160 bytes 2026-02-22T21:48:19.573 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7400 IOs, 3106371584 bytes 2026-02-22T21:48:19.625 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7500 IOs, 3176168448 bytes 2026-02-22T21:48:19.669 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7600 IOs, 3192623104 bytes 2026-02-22T21:48:19.703 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7700 IOs, 3206091776 bytes 2026-02-22T21:48:19.753 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7800 IOs, 3252151296 bytes 2026-02-22T21:48:19.784 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7900 IOs, 3265810432 bytes 2026-02-22T21:48:19.811 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8000 IOs, 3266465792 bytes 2026-02-22T21:48:19.870 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8100 IOs, 3361558528 bytes 2026-02-22T21:48:19.930 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8200 IOs, 3430726656 bytes 2026-02-22T21:48:19.975 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8300 IOs, 3464702976 bytes 2026-02-22T21:48:20.015 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8400 IOs, 3500958720 bytes 2026-02-22T21:48:20.091 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8500 IOs, 3591539712 bytes 2026-02-22T21:48:20.155 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8600 IOs, 3668835328 bytes 2026-02-22T21:48:20.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8700 IOs, 3692335104 bytes 2026-02-22T21:48:20.229 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8800 IOs, 3716578304 bytes 2026-02-22T21:48:20.294 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8900 IOs, 3791749120 bytes 2026-02-22T21:48:20.335 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9000 IOs, 3823703040 bytes 2026-02-22T21:48:20.386 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9100 IOs, 3877629952 bytes 2026-02-22T21:48:20.437 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9200 IOs, 3920886784 bytes 2026-02-22T21:48:20.482 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9300 IOs, 3969366016 bytes 2026-02-22T21:48:20.540 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9400 IOs, 4045197312 bytes 2026-02-22T21:48:20.588 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9500 IOs, 4111319040 bytes 2026-02-22T21:48:20.632 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9600 IOs, 4131994624 bytes 2026-02-22T21:48:20.663 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9700 IOs, 4157757440 bytes 2026-02-22T21:48:20.701 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9800 IOs, 4192136192 bytes 2026-02-22T21:48:20.751 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9900 IOs, 4231360512 bytes 2026-02-22T21:48:20.813 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10000 IOs, 4297293824 bytes 2026-02-22T21:48:20.855 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10100 IOs, 4335774720 bytes 2026-02-22T21:48:20.886 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10200 IOs, 4336354304 bytes 2026-02-22T21:48:20.920 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10300 IOs, 4352185344 bytes 2026-02-22T21:48:20.960 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10400 IOs, 4378196992 bytes 2026-02-22T21:48:21.002 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10500 IOs, 4412658688 bytes 2026-02-22T21:48:21.050 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10600 IOs, 4438132736 bytes 2026-02-22T21:48:21.091 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10700 IOs, 4465449984 bytes 2026-02-22T21:48:21.134 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10800 IOs, 4504722432 bytes 2026-02-22T21:48:21.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ error_code=124 2026-02-22T21:48:21.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set -e 2026-02-22T21:48:21.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' 124 -eq 124 ']' 2026-02-22T21:48:21.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:48:21.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T21:48:21.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_4 2026-02-22T21:48:21.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image_and_enable_mirror cluster2 mirror image_4 snapshot 128M 2026-02-22T21:48:21.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:48:21.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:48:21.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:48:21.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:48:21.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_4 2026-02-22T21:48:21.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:48:21.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T21:48:21.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n snapshot ']' 2026-02-22T21:48:21.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:48:21.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image cluster2 mirror image_4 128M 2026-02-22T21:48:21.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:48:21.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:48:21.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:48:21.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:48:21.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_4 2026-02-22T21:48:21.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:48:21.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local size=128 2026-02-22T21:48:21.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n 128M ']' 2026-02-22T21:48:21.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ size=128M 2026-02-22T21:48:21.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:48:21.180 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_4 2026-02-22T21:48:21.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T21:48:21.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:48:21.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' image = image ']' 2026-02-22T21:48:21.209 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:48:21.209 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ enable_mirror cluster2 mirror image_4 snapshot 2026-02-22T21:48:21.209 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:48:21.209 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:48:21.209 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_4 2026-02-22T21:48:21.209 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T21:48:21.209 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_4 snapshot 2026-02-22T21:48:22.149 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Mirroring enabled 2026-02-22T21:48:22.154 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 info mirror/image_4 2026-02-22T21:48:22.180 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_4': 2026-02-22T21:48:22.180 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 128 MiB in 32 objects 2026-02-22T21:48:22.180 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T21:48:22.180 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 1 2026-02-22T21:48:22.180 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 12961c5cda9f 2026-02-22T21:48:22.180 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.12961c5cda9f 2026-02-22T21:48:22.180 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T21:48:22.180 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock 2026-02-22T21:48:22.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T21:48:22.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T21:48:22.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:48:21 2026 2026-02-22T21:48:22.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:48:21 2026 2026-02-22T21:48:22.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:48:21 2026 2026-02-22T21:48:22.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T21:48:22.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T21:48:22.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: 23c3bee2-bece-4830-b190-6b5f39e55cb1 2026-02-22T21:48:22.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: true 2026-02-22T21:48:22.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n '' ']' 2026-02-22T21:48:22.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ stress_write_image cluster2 mirror image_4 2026-02-22T21:48:22.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:48:22.183 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:48:22.183 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_4 2026-02-22T21:48:22.183 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-22T21:48:22.183 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local duration=13 2026-02-22T21:48:22.183 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set +e 2026-02-22T21:48:22.184 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ timeout 13s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_4 --debug-rbd=20 --debug-journaler=20 2026-02-22T21:48:22.252 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:100 IOs, 24844288 bytes 2026-02-22T21:48:22.291 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:200 IOs, 35341312 bytes 2026-02-22T21:48:22.355 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:300 IOs, 127762432 bytes 2026-02-22T21:48:22.411 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:400 IOs, 214283264 bytes 2026-02-22T21:48:22.462 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:500 IOs, 256662528 bytes 2026-02-22T21:48:22.512 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:600 IOs, 306786304 bytes 2026-02-22T21:48:22.537 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:700 IOs, 307427328 bytes 2026-02-22T21:48:22.584 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:800 IOs, 376208384 bytes 2026-02-22T21:48:22.629 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:900 IOs, 419054592 bytes 2026-02-22T21:48:22.665 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1000 IOs, 422052864 bytes 2026-02-22T21:48:22.713 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1100 IOs, 479073280 bytes 2026-02-22T21:48:22.754 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1200 IOs, 501209088 bytes 2026-02-22T21:48:22.792 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1300 IOs, 526320640 bytes 2026-02-22T21:48:22.832 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1400 IOs, 548851712 bytes 2026-02-22T21:48:22.867 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1500 IOs, 570051584 bytes 2026-02-22T21:48:22.901 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1600 IOs, 586346496 bytes 2026-02-22T21:48:22.958 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1700 IOs, 658277376 bytes 2026-02-22T21:48:23.000 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1800 IOs, 676741120 bytes 2026-02-22T21:48:23.039 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1900 IOs, 717949952 bytes 2026-02-22T21:48:23.089 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2000 IOs, 745166848 bytes 2026-02-22T21:48:23.128 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2100 IOs, 776022016 bytes 2026-02-22T21:48:23.195 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2200 IOs, 844514304 bytes 2026-02-22T21:48:23.226 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2300 IOs, 860531712 bytes 2026-02-22T21:48:23.262 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2400 IOs, 881134592 bytes 2026-02-22T21:48:23.304 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2500 IOs, 936297472 bytes 2026-02-22T21:48:23.339 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2600 IOs, 944641024 bytes 2026-02-22T21:48:23.395 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2700 IOs, 1007654912 bytes 2026-02-22T21:48:23.446 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2800 IOs, 1062744064 bytes 2026-02-22T21:48:23.496 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2900 IOs, 1121755136 bytes 2026-02-22T21:48:23.556 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3000 IOs, 1197475840 bytes 2026-02-22T21:48:23.600 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3100 IOs, 1221124096 bytes 2026-02-22T21:48:23.629 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3200 IOs, 1232248832 bytes 2026-02-22T21:48:23.670 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3300 IOs, 1266964480 bytes 2026-02-22T21:48:23.699 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3400 IOs, 1285444608 bytes 2026-02-22T21:48:23.747 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3500 IOs, 1328485376 bytes 2026-02-22T21:48:23.806 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3600 IOs, 1388040192 bytes 2026-02-22T21:48:23.839 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3700 IOs, 1401072640 bytes 2026-02-22T21:48:23.878 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3800 IOs, 1430323200 bytes 2026-02-22T21:48:23.924 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3900 IOs, 1478430720 bytes 2026-02-22T21:48:23.958 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4000 IOs, 1492495360 bytes 2026-02-22T21:48:24.017 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4100 IOs, 1569942528 bytes 2026-02-22T21:48:24.064 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4200 IOs, 1620886528 bytes 2026-02-22T21:48:24.094 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4300 IOs, 1629358080 bytes 2026-02-22T21:48:24.163 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4400 IOs, 1729340416 bytes 2026-02-22T21:48:24.229 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4500 IOs, 1809224704 bytes 2026-02-22T21:48:24.265 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4600 IOs, 1828508672 bytes 2026-02-22T21:48:24.321 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4700 IOs, 1897103360 bytes 2026-02-22T21:48:24.364 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4800 IOs, 1948290048 bytes 2026-02-22T21:48:24.411 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4900 IOs, 1972167680 bytes 2026-02-22T21:48:24.449 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5000 IOs, 1997761536 bytes 2026-02-22T21:48:24.507 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5100 IOs, 2067176448 bytes 2026-02-22T21:48:24.556 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5200 IOs, 2119530496 bytes 2026-02-22T21:48:24.587 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5300 IOs, 2126346240 bytes 2026-02-22T21:48:24.615 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5400 IOs, 2147680256 bytes 2026-02-22T21:48:24.655 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5500 IOs, 2163304448 bytes 2026-02-22T21:48:24.693 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5600 IOs, 2184877056 bytes 2026-02-22T21:48:24.734 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5700 IOs, 2230352896 bytes 2026-02-22T21:48:24.778 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5800 IOs, 2255692800 bytes 2026-02-22T21:48:24.819 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5900 IOs, 2288726016 bytes 2026-02-22T21:48:24.867 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6000 IOs, 2345066496 bytes 2026-02-22T21:48:24.908 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6100 IOs, 2374614016 bytes 2026-02-22T21:48:24.969 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6200 IOs, 2482819072 bytes 2026-02-22T21:48:25.008 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6300 IOs, 2495083520 bytes 2026-02-22T21:48:25.048 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6400 IOs, 2525390848 bytes 2026-02-22T21:48:25.099 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6500 IOs, 2574196736 bytes 2026-02-22T21:48:25.138 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6600 IOs, 2595319808 bytes 2026-02-22T21:48:25.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6700 IOs, 2659283968 bytes 2026-02-22T21:48:25.277 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6800 IOs, 2780135424 bytes 2026-02-22T21:48:25.317 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6900 IOs, 2800770048 bytes 2026-02-22T21:48:25.345 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7000 IOs, 2801381376 bytes 2026-02-22T21:48:25.392 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7100 IOs, 2858128384 bytes 2026-02-22T21:48:25.431 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7200 IOs, 2866028544 bytes 2026-02-22T21:48:25.469 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7300 IOs, 2888977408 bytes 2026-02-22T21:48:25.498 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7400 IOs, 2890360832 bytes 2026-02-22T21:48:25.539 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7500 IOs, 2912844800 bytes 2026-02-22T21:48:25.583 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7600 IOs, 2943560704 bytes 2026-02-22T21:48:25.628 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7700 IOs, 2997690368 bytes 2026-02-22T21:48:25.670 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7800 IOs, 3018052608 bytes 2026-02-22T21:48:25.696 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7900 IOs, 3018670080 bytes 2026-02-22T21:48:25.744 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8000 IOs, 3083429888 bytes 2026-02-22T21:48:25.807 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8100 IOs, 3165045760 bytes 2026-02-22T21:48:25.839 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8200 IOs, 3195714560 bytes 2026-02-22T21:48:25.895 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8300 IOs, 3252514816 bytes 2026-02-22T21:48:25.984 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8400 IOs, 3408096256 bytes 2026-02-22T21:48:26.039 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8500 IOs, 3486538752 bytes 2026-02-22T21:48:26.097 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8600 IOs, 3532136448 bytes 2026-02-22T21:48:26.148 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8700 IOs, 3608088576 bytes 2026-02-22T21:48:26.214 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8800 IOs, 3685762048 bytes 2026-02-22T21:48:26.263 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8900 IOs, 3725524992 bytes 2026-02-22T21:48:26.297 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9000 IOs, 3744576512 bytes 2026-02-22T21:48:26.332 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9100 IOs, 3763563520 bytes 2026-02-22T21:48:26.391 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9200 IOs, 3841351680 bytes 2026-02-22T21:48:26.439 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9300 IOs, 3894580224 bytes 2026-02-22T21:48:26.488 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9400 IOs, 3939914752 bytes 2026-02-22T21:48:26.543 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9500 IOs, 4011801600 bytes 2026-02-22T21:48:26.579 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9600 IOs, 4034082816 bytes 2026-02-22T21:48:26.611 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9700 IOs, 4049213440 bytes 2026-02-22T21:48:26.660 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9800 IOs, 4100580352 bytes 2026-02-22T21:48:26.698 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9900 IOs, 4119086080 bytes 2026-02-22T21:48:26.756 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10000 IOs, 4192402432 bytes 2026-02-22T21:48:26.801 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10100 IOs, 4244568064 bytes 2026-02-22T21:48:26.845 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10200 IOs, 4271008768 bytes 2026-02-22T21:48:26.882 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10300 IOs, 4306683904 bytes 2026-02-22T21:48:26.931 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10400 IOs, 4335238144 bytes 2026-02-22T21:48:26.968 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10500 IOs, 4352794624 bytes 2026-02-22T21:48:27.014 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10600 IOs, 4390838272 bytes 2026-02-22T21:48:27.062 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10700 IOs, 4436498432 bytes 2026-02-22T21:48:27.117 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10800 IOs, 4476756992 bytes 2026-02-22T21:48:27.146 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10900 IOs, 4480231424 bytes 2026-02-22T21:48:27.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11000 IOs, 4488675328 bytes 2026-02-22T21:48:27.236 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11100 IOs, 4548085760 bytes 2026-02-22T21:48:27.282 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11200 IOs, 4583187456 bytes 2026-02-22T21:48:27.357 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11300 IOs, 4699133952 bytes 2026-02-22T21:48:27.410 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11400 IOs, 4755165184 bytes 2026-02-22T21:48:27.460 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11500 IOs, 4832625664 bytes 2026-02-22T21:48:27.499 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11600 IOs, 4850250752 bytes 2026-02-22T21:48:27.544 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11700 IOs, 4892053504 bytes 2026-02-22T21:48:27.575 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11800 IOs, 4904160256 bytes 2026-02-22T21:48:27.636 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11900 IOs, 4975319040 bytes 2026-02-22T21:48:27.682 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12000 IOs, 5020203008 bytes 2026-02-22T21:48:27.724 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12100 IOs, 5057982464 bytes 2026-02-22T21:48:27.764 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12200 IOs, 5079947264 bytes 2026-02-22T21:48:27.809 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12300 IOs, 5118948352 bytes 2026-02-22T21:48:27.842 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12400 IOs, 5142424576 bytes 2026-02-22T21:48:27.894 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12500 IOs, 5176292352 bytes 2026-02-22T21:48:27.949 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12600 IOs, 5249797120 bytes 2026-02-22T21:48:27.984 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12700 IOs, 5256628224 bytes 2026-02-22T21:48:28.020 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12800 IOs, 5290292224 bytes 2026-02-22T21:48:28.067 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12900 IOs, 5333286912 bytes 2026-02-22T21:48:28.099 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13000 IOs, 5341393920 bytes 2026-02-22T21:48:28.154 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13100 IOs, 5384735744 bytes 2026-02-22T21:48:28.185 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13200 IOs, 5393439744 bytes 2026-02-22T21:48:28.218 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13300 IOs, 5399941120 bytes 2026-02-22T21:48:28.265 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13400 IOs, 5444632576 bytes 2026-02-22T21:48:28.309 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13500 IOs, 5479122944 bytes 2026-02-22T21:48:28.351 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13600 IOs, 5508855808 bytes 2026-02-22T21:48:28.399 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13700 IOs, 5541348352 bytes 2026-02-22T21:48:28.450 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13800 IOs, 5584634880 bytes 2026-02-22T21:48:28.493 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13900 IOs, 5615141888 bytes 2026-02-22T21:48:28.540 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14000 IOs, 5663329280 bytes 2026-02-22T21:48:28.588 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14100 IOs, 5701058560 bytes 2026-02-22T21:48:28.629 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14200 IOs, 5726658560 bytes 2026-02-22T21:48:28.656 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14300 IOs, 5732847616 bytes 2026-02-22T21:48:28.695 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14400 IOs, 5758032896 bytes 2026-02-22T21:48:28.762 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14500 IOs, 5861750784 bytes 2026-02-22T21:48:28.827 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14600 IOs, 5958477824 bytes 2026-02-22T21:48:28.885 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14700 IOs, 6034495488 bytes 2026-02-22T21:48:28.962 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14800 IOs, 6154007552 bytes 2026-02-22T21:48:29.009 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14900 IOs, 6206514176 bytes 2026-02-22T21:48:29.063 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15000 IOs, 6259564544 bytes 2026-02-22T21:48:29.096 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15100 IOs, 6285532160 bytes 2026-02-22T21:48:29.144 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15200 IOs, 6312647680 bytes 2026-02-22T21:48:29.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15300 IOs, 6360010752 bytes 2026-02-22T21:48:29.245 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15400 IOs, 6396174336 bytes 2026-02-22T21:48:29.294 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15500 IOs, 6443991040 bytes 2026-02-22T21:48:29.345 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15600 IOs, 6497463296 bytes 2026-02-22T21:48:29.410 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15700 IOs, 6606131200 bytes 2026-02-22T21:48:29.433 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.8492409437793482) > weight (0.25) 2026-02-22T21:48:29.433 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.5537734533085534) > weight (0.25) 2026-02-22T21:48:29.433 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.2 2026-02-22T21:48:29.434 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Sent signal 15 2026-02-22T21:48:29.434 INFO:tasks.rbd_mirror_thrash:waiting for 8.170461157520561 secs before reviving daemons 2026-02-22T21:48:29.434 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial026.stderr:2026-02-22T21:48:29.433+0000 7ff342ae6640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.2 (PID: 51434) UID: 1000 2026-02-22T21:48:29.458 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15800 IOs, 6637800448 bytes 2026-02-22T21:48:29.492 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15900 IOs, 6646605824 bytes 2026-02-22T21:48:29.519 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16000 IOs, 6647184384 bytes 2026-02-22T21:48:29.570 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16100 IOs, 6705494016 bytes 2026-02-22T21:48:29.620 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16200 IOs, 6757775360 bytes 2026-02-22T21:48:29.667 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16300 IOs, 6804321280 bytes 2026-02-22T21:48:29.708 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16400 IOs, 6833525760 bytes 2026-02-22T21:48:29.765 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16500 IOs, 6904046592 bytes 2026-02-22T21:48:29.803 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16600 IOs, 6921477120 bytes 2026-02-22T21:48:29.828 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16700 IOs, 6922095616 bytes 2026-02-22T21:48:29.889 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16800 IOs, 7018760192 bytes 2026-02-22T21:48:29.945 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16900 IOs, 7088900096 bytes 2026-02-22T21:48:29.996 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17000 IOs, 7136309248 bytes 2026-02-22T21:48:30.025 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17100 IOs, 7141793792 bytes 2026-02-22T21:48:30.067 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17200 IOs, 7177976832 bytes 2026-02-22T21:48:30.108 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17300 IOs, 7218294784 bytes 2026-02-22T21:48:30.144 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17400 IOs, 7218919424 bytes 2026-02-22T21:48:30.185 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17500 IOs, 7247243264 bytes 2026-02-22T21:48:30.236 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17600 IOs, 7297356800 bytes 2026-02-22T21:48:30.276 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17700 IOs, 7327473664 bytes 2026-02-22T21:48:30.321 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17800 IOs, 7363768320 bytes 2026-02-22T21:48:30.360 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17900 IOs, 7381943296 bytes 2026-02-22T21:48:30.408 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18000 IOs, 7448467456 bytes 2026-02-22T21:48:30.448 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18100 IOs, 7457712128 bytes 2026-02-22T21:48:30.495 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18200 IOs, 7516001280 bytes 2026-02-22T21:48:30.539 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18300 IOs, 7537265664 bytes 2026-02-22T21:48:30.581 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18400 IOs, 7566672896 bytes 2026-02-22T21:48:30.608 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18500 IOs, 7567467520 bytes 2026-02-22T21:48:30.652 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18600 IOs, 7606747136 bytes 2026-02-22T21:48:30.681 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18700 IOs, 7612304384 bytes 2026-02-22T21:48:30.725 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18800 IOs, 7657210880 bytes 2026-02-22T21:48:30.763 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18900 IOs, 7680565248 bytes 2026-02-22T21:48:30.793 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19000 IOs, 7685908480 bytes 2026-02-22T21:48:30.842 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19100 IOs, 7734103040 bytes 2026-02-22T21:48:30.890 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19200 IOs, 7781535744 bytes 2026-02-22T21:48:30.932 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19300 IOs, 7823890432 bytes 2026-02-22T21:48:30.977 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19400 IOs, 7858696192 bytes 2026-02-22T21:48:31.014 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19500 IOs, 7905809408 bytes 2026-02-22T21:48:31.083 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19600 IOs, 7990581248 bytes 2026-02-22T21:48:31.138 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19700 IOs, 8037929984 bytes 2026-02-22T21:48:31.171 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19800 IOs, 8046481408 bytes 2026-02-22T21:48:31.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19900 IOs, 8091893760 bytes 2026-02-22T21:48:31.264 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20000 IOs, 8128210944 bytes 2026-02-22T21:48:31.297 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20100 IOs, 8158840832 bytes 2026-02-22T21:48:31.346 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20200 IOs, 8201282560 bytes 2026-02-22T21:48:31.393 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20300 IOs, 8250954752 bytes 2026-02-22T21:48:31.435 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20400 IOs, 8291774464 bytes 2026-02-22T21:48:31.488 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20500 IOs, 8332230656 bytes 2026-02-22T21:48:31.529 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20600 IOs, 8359369728 bytes 2026-02-22T21:48:31.557 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20700 IOs, 8372135936 bytes 2026-02-22T21:48:31.604 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20800 IOs, 8402834432 bytes 2026-02-22T21:48:31.649 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20900 IOs, 8439112704 bytes 2026-02-22T21:48:31.677 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21000 IOs, 8439712768 bytes 2026-02-22T21:48:31.719 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21100 IOs, 8468553728 bytes 2026-02-22T21:48:31.755 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21200 IOs, 8503862272 bytes 2026-02-22T21:48:31.798 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21300 IOs, 8532673536 bytes 2026-02-22T21:48:31.848 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21400 IOs, 8587718656 bytes 2026-02-22T21:48:31.882 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21500 IOs, 8595041280 bytes 2026-02-22T21:48:31.923 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21600 IOs, 8630886400 bytes 2026-02-22T21:48:31.961 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21700 IOs, 8660699136 bytes 2026-02-22T21:48:32.015 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21800 IOs, 8720569344 bytes 2026-02-22T21:48:32.069 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21900 IOs, 8772841472 bytes 2026-02-22T21:48:32.097 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22000 IOs, 8773459968 bytes 2026-02-22T21:48:32.149 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22100 IOs, 8825528320 bytes 2026-02-22T21:48:32.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22200 IOs, 8869607424 bytes 2026-02-22T21:48:32.227 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22300 IOs, 8903911424 bytes 2026-02-22T21:48:32.280 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22400 IOs, 8939777024 bytes 2026-02-22T21:48:32.339 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22500 IOs, 9010129920 bytes 2026-02-22T21:48:32.388 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22600 IOs, 9049090048 bytes 2026-02-22T21:48:32.442 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22700 IOs, 9120020480 bytes 2026-02-22T21:48:32.507 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22800 IOs, 9210419200 bytes 2026-02-22T21:48:32.535 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22900 IOs, 9229325312 bytes 2026-02-22T21:48:32.577 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23000 IOs, 9255860224 bytes 2026-02-22T21:48:32.639 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23100 IOs, 9326412800 bytes 2026-02-22T21:48:32.684 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23200 IOs, 9372438528 bytes 2026-02-22T21:48:32.719 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23300 IOs, 9381521408 bytes 2026-02-22T21:48:32.747 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23400 IOs, 9383514112 bytes 2026-02-22T21:48:32.798 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23500 IOs, 9441176576 bytes 2026-02-22T21:48:32.840 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23600 IOs, 9462423552 bytes 2026-02-22T21:48:32.878 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23700 IOs, 9480209408 bytes 2026-02-22T21:48:32.944 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23800 IOs, 9581500416 bytes 2026-02-22T21:48:32.992 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23900 IOs, 9634307072 bytes 2026-02-22T21:48:33.040 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24000 IOs, 9658156032 bytes 2026-02-22T21:48:33.077 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24100 IOs, 9683505152 bytes 2026-02-22T21:48:33.131 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24200 IOs, 9735718912 bytes 2026-02-22T21:48:33.195 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24300 IOs, 9779084288 bytes 2026-02-22T21:48:33.239 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24400 IOs, 9808194560 bytes 2026-02-22T21:48:33.286 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24500 IOs, 9851311104 bytes 2026-02-22T21:48:33.343 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24600 IOs, 9904881664 bytes 2026-02-22T21:48:33.386 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24700 IOs, 9947639808 bytes 2026-02-22T21:48:33.446 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24800 IOs, 10013012992 bytes 2026-02-22T21:48:33.480 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24900 IOs, 10037027840 bytes 2026-02-22T21:48:33.511 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25000 IOs, 10037655552 bytes 2026-02-22T21:48:33.553 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25100 IOs, 10074902528 bytes 2026-02-22T21:48:33.611 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25200 IOs, 10143741952 bytes 2026-02-22T21:48:33.660 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25300 IOs, 10226497536 bytes 2026-02-22T21:48:33.713 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25400 IOs, 10254886912 bytes 2026-02-22T21:48:33.759 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25500 IOs, 10299337728 bytes 2026-02-22T21:48:33.816 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25600 IOs, 10364625920 bytes 2026-02-22T21:48:33.868 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25700 IOs, 10420810752 bytes 2026-02-22T21:48:33.937 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25800 IOs, 10518413312 bytes 2026-02-22T21:48:33.991 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25900 IOs, 10579911680 bytes 2026-02-22T21:48:34.054 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:26000 IOs, 10660136960 bytes 2026-02-22T21:48:34.082 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:26100 IOs, 10666362880 bytes 2026-02-22T21:48:34.113 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:26200 IOs, 10694161408 bytes 2026-02-22T21:48:34.157 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:26300 IOs, 10723525632 bytes 2026-02-22T21:48:34.188 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:26400 IOs, 10724123648 bytes 2026-02-22T21:48:34.233 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:26500 IOs, 10787665920 bytes 2026-02-22T21:48:34.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:26600 IOs, 10799941632 bytes 2026-02-22T21:48:34.324 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:26700 IOs, 10827290624 bytes 2026-02-22T21:48:34.372 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:26800 IOs, 10872279040 bytes 2026-02-22T21:48:34.423 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:26900 IOs, 10915940352 bytes 2026-02-22T21:48:34.477 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:27000 IOs, 10970824704 bytes 2026-02-22T21:48:34.522 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:27100 IOs, 11003576320 bytes 2026-02-22T21:48:34.559 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:27200 IOs, 11020544000 bytes 2026-02-22T21:48:34.602 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:27300 IOs, 11054701568 bytes 2026-02-22T21:48:34.637 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:27400 IOs, 11086611456 bytes 2026-02-22T21:48:34.670 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:27500 IOs, 11087596544 bytes 2026-02-22T21:48:34.702 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:27600 IOs, 11095897088 bytes 2026-02-22T21:48:34.931 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:27700 IOs, 11096487936 bytes 2026-02-22T21:48:34.959 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:27800 IOs, 11097593856 bytes 2026-02-22T21:48:35.002 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:27900 IOs, 11154202624 bytes 2026-02-22T21:48:35.044 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:28000 IOs, 11169002496 bytes 2026-02-22T21:48:35.092 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:28100 IOs, 11222566912 bytes 2026-02-22T21:48:35.137 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:28200 IOs, 11260859392 bytes 2026-02-22T21:48:35.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ error_code=124 2026-02-22T21:48:35.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set -e 2026-02-22T21:48:35.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' 124 -eq 124 ']' 2026-02-22T21:48:35.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:48:35.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T21:48:35.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_5 2026-02-22T21:48:35.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image_and_enable_mirror cluster2 mirror image_5 snapshot 128M 2026-02-22T21:48:35.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:48:35.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:48:35.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:48:35.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:48:35.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_5 2026-02-22T21:48:35.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:48:35.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T21:48:35.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n snapshot ']' 2026-02-22T21:48:35.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:48:35.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image cluster2 mirror image_5 128M 2026-02-22T21:48:35.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:48:35.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:48:35.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:48:35.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:48:35.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_5 2026-02-22T21:48:35.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:48:35.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local size=128 2026-02-22T21:48:35.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n 128M ']' 2026-02-22T21:48:35.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ size=128M 2026-02-22T21:48:35.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:48:35.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_5 2026-02-22T21:48:35.220 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T21:48:35.220 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:48:35.220 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' image = image ']' 2026-02-22T21:48:35.220 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:48:35.220 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ enable_mirror cluster2 mirror image_5 snapshot 2026-02-22T21:48:35.220 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:48:35.220 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:48:35.220 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_5 2026-02-22T21:48:35.220 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T21:48:35.220 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_5 snapshot 2026-02-22T21:48:36.087 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Mirroring enabled 2026-02-22T21:48:36.092 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 info mirror/image_5 2026-02-22T21:48:36.116 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_5': 2026-02-22T21:48:36.116 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 128 MiB in 32 objects 2026-02-22T21:48:36.116 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T21:48:36.116 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 1 2026-02-22T21:48:36.116 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 12a0a8911b47 2026-02-22T21:48:36.117 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.12a0a8911b47 2026-02-22T21:48:36.117 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T21:48:36.117 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock 2026-02-22T21:48:36.117 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T21:48:36.117 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T21:48:36.117 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:48:35 2026 2026-02-22T21:48:36.117 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:48:35 2026 2026-02-22T21:48:36.117 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:48:35 2026 2026-02-22T21:48:36.117 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T21:48:36.117 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T21:48:36.117 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: d7359255-a2a8-4833-b9d4-992b53784bf0 2026-02-22T21:48:36.117 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: true 2026-02-22T21:48:36.118 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n '' ']' 2026-02-22T21:48:36.118 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ stress_write_image cluster2 mirror image_5 2026-02-22T21:48:36.118 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:48:36.119 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:48:36.119 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_5 2026-02-22T21:48:36.119 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-22T21:48:36.119 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local duration=7 2026-02-22T21:48:36.119 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set +e 2026-02-22T21:48:36.120 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ timeout 7s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_5 --debug-rbd=20 --debug-journaler=20 2026-02-22T21:48:36.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:100 IOs, 47387648 bytes 2026-02-22T21:48:36.245 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:200 IOs, 89236480 bytes 2026-02-22T21:48:36.292 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:300 IOs, 138236928 bytes 2026-02-22T21:48:36.322 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:400 IOs, 149042176 bytes 2026-02-22T21:48:36.364 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:500 IOs, 190499840 bytes 2026-02-22T21:48:36.408 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:600 IOs, 245668864 bytes 2026-02-22T21:48:36.457 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:700 IOs, 301346816 bytes 2026-02-22T21:48:36.492 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:800 IOs, 328088576 bytes 2026-02-22T21:48:36.529 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:900 IOs, 347342848 bytes 2026-02-22T21:48:36.567 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1000 IOs, 384056320 bytes 2026-02-22T21:48:36.604 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1100 IOs, 422799360 bytes 2026-02-22T21:48:36.678 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1200 IOs, 541824000 bytes 2026-02-22T21:48:36.730 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1300 IOs, 588675072 bytes 2026-02-22T21:48:36.778 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1400 IOs, 643135488 bytes 2026-02-22T21:48:36.817 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1500 IOs, 673970176 bytes 2026-02-22T21:48:36.887 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1600 IOs, 765755392 bytes 2026-02-22T21:48:36.931 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1700 IOs, 817771520 bytes 2026-02-22T21:48:36.979 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1800 IOs, 869313536 bytes 2026-02-22T21:48:37.027 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1900 IOs, 936287232 bytes 2026-02-22T21:48:37.069 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2000 IOs, 963701760 bytes 2026-02-22T21:48:37.136 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2100 IOs, 1069075456 bytes 2026-02-22T21:48:37.171 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2200 IOs, 1100064768 bytes 2026-02-22T21:48:37.209 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2300 IOs, 1119126528 bytes 2026-02-22T21:48:37.270 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2400 IOs, 1194740736 bytes 2026-02-22T21:48:37.329 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2500 IOs, 1273794560 bytes 2026-02-22T21:48:37.376 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2600 IOs, 1323880448 bytes 2026-02-22T21:48:37.404 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2700 IOs, 1339463680 bytes 2026-02-22T21:48:37.455 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2800 IOs, 1386369024 bytes 2026-02-22T21:48:37.510 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2900 IOs, 1441649664 bytes 2026-02-22T21:48:37.553 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3000 IOs, 1491155968 bytes 2026-02-22T21:48:37.605 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.2 2026-02-22T21:48:37.605 INFO:teuthology.orchestra.run:waiting for 600 2026-02-22T21:48:37.605 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.2 2026-02-22T21:48:37.605 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Restarting daemon 2026-02-22T21:48:37.605 DEBUG:teuthology.orchestra.run.trial026:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.2 2026-02-22T21:48:37.608 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Started 2026-02-22T21:48:37.608 INFO:tasks.rbd_mirror_thrash:waiting for 103.55303744745657 secs before thrashing 2026-02-22T21:48:37.615 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3100 IOs, 1612496896 bytes 2026-02-22T21:48:37.670 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3200 IOs, 1636666368 bytes 2026-02-22T21:48:37.740 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3300 IOs, 1733430272 bytes 2026-02-22T21:48:37.788 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3400 IOs, 1773649920 bytes 2026-02-22T21:48:37.842 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3500 IOs, 1846053888 bytes 2026-02-22T21:48:37.896 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3600 IOs, 1908704256 bytes 2026-02-22T21:48:37.954 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3700 IOs, 1981014016 bytes 2026-02-22T21:48:37.996 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3800 IOs, 2016071680 bytes 2026-02-22T21:48:38.044 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3900 IOs, 2045989888 bytes 2026-02-22T21:48:38.076 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4000 IOs, 2066357248 bytes 2026-02-22T21:48:38.111 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4100 IOs, 2079710208 bytes 2026-02-22T21:48:38.143 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4200 IOs, 2090686464 bytes 2026-02-22T21:48:38.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4300 IOs, 2141927424 bytes 2026-02-22T21:48:38.241 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4400 IOs, 2173901824 bytes 2026-02-22T21:48:38.273 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4500 IOs, 2187356160 bytes 2026-02-22T21:48:38.329 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4600 IOs, 2234469376 bytes 2026-02-22T21:48:38.368 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4700 IOs, 2267993088 bytes 2026-02-22T21:48:38.420 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4800 IOs, 2340579328 bytes 2026-02-22T21:48:38.475 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4900 IOs, 2394953728 bytes 2026-02-22T21:48:38.517 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5000 IOs, 2428882944 bytes 2026-02-22T21:48:38.553 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5100 IOs, 2459373568 bytes 2026-02-22T21:48:38.594 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5200 IOs, 2485223424 bytes 2026-02-22T21:48:38.652 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5300 IOs, 2578721792 bytes 2026-02-22T21:48:38.708 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5400 IOs, 2635227136 bytes 2026-02-22T21:48:38.745 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5500 IOs, 2656496640 bytes 2026-02-22T21:48:38.783 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5600 IOs, 2688809984 bytes 2026-02-22T21:48:38.848 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5700 IOs, 2783563776 bytes 2026-02-22T21:48:38.883 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5800 IOs, 2806003712 bytes 2026-02-22T21:48:38.935 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5900 IOs, 2874662912 bytes 2026-02-22T21:48:38.998 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6000 IOs, 2940043264 bytes 2026-02-22T21:48:39.033 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6100 IOs, 2960159744 bytes 2026-02-22T21:48:39.078 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6200 IOs, 3018916864 bytes 2026-02-22T21:48:39.130 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6300 IOs, 3078994944 bytes 2026-02-22T21:48:39.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6400 IOs, 3113674752 bytes 2026-02-22T21:48:39.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6500 IOs, 3144705024 bytes 2026-02-22T21:48:39.252 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6600 IOs, 3176360960 bytes 2026-02-22T21:48:39.307 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6700 IOs, 3222630400 bytes 2026-02-22T21:48:39.356 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6800 IOs, 3260987392 bytes 2026-02-22T21:48:39.401 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6900 IOs, 3294204928 bytes 2026-02-22T21:48:39.441 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7000 IOs, 3332532224 bytes 2026-02-22T21:48:39.492 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7100 IOs, 3383400448 bytes 2026-02-22T21:48:39.531 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7200 IOs, 3413582848 bytes 2026-02-22T21:48:39.588 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7300 IOs, 3498707968 bytes 2026-02-22T21:48:39.623 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7400 IOs, 3525909504 bytes 2026-02-22T21:48:39.659 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7500 IOs, 3537806336 bytes 2026-02-22T21:48:39.691 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7600 IOs, 3541484544 bytes 2026-02-22T21:48:39.737 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7700 IOs, 3590775808 bytes 2026-02-22T21:48:39.766 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7800 IOs, 3597088768 bytes 2026-02-22T21:48:39.797 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7900 IOs, 3621896192 bytes 2026-02-22T21:48:39.846 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8000 IOs, 3655959552 bytes 2026-02-22T21:48:39.891 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8100 IOs, 3679561728 bytes 2026-02-22T21:48:39.956 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8200 IOs, 3761994752 bytes 2026-02-22T21:48:40.000 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8300 IOs, 3801254912 bytes 2026-02-22T21:48:40.049 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8400 IOs, 3854956544 bytes 2026-02-22T21:48:40.089 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8500 IOs, 3883841536 bytes 2026-02-22T21:48:40.147 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8600 IOs, 3959069696 bytes 2026-02-22T21:48:40.186 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8700 IOs, 3986654208 bytes 2026-02-22T21:48:40.225 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8800 IOs, 4016900096 bytes 2026-02-22T21:48:40.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8900 IOs, 4065393664 bytes 2026-02-22T21:48:40.336 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9000 IOs, 4131478528 bytes 2026-02-22T21:48:40.378 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9100 IOs, 4165944320 bytes 2026-02-22T21:48:40.427 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9200 IOs, 4205497344 bytes 2026-02-22T21:48:40.454 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9300 IOs, 4210262016 bytes 2026-02-22T21:48:40.480 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9400 IOs, 4210865152 bytes 2026-02-22T21:48:40.520 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9500 IOs, 4234525696 bytes 2026-02-22T21:48:40.568 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9600 IOs, 4292175872 bytes 2026-02-22T21:48:40.608 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9700 IOs, 4338399232 bytes 2026-02-22T21:48:40.647 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9800 IOs, 4362049536 bytes 2026-02-22T21:48:40.686 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9900 IOs, 4389702656 bytes 2026-02-22T21:48:40.730 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10000 IOs, 4430921728 bytes 2026-02-22T21:48:40.765 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10100 IOs, 4452750336 bytes 2026-02-22T21:48:40.823 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10200 IOs, 4519838720 bytes 2026-02-22T21:48:40.863 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10300 IOs, 4554349568 bytes 2026-02-22T21:48:40.928 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10400 IOs, 4638012416 bytes 2026-02-22T21:48:40.954 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10500 IOs, 4638583808 bytes 2026-02-22T21:48:40.993 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10600 IOs, 4665929728 bytes 2026-02-22T21:48:41.057 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10700 IOs, 4752087040 bytes 2026-02-22T21:48:41.087 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10800 IOs, 4752709632 bytes 2026-02-22T21:48:41.134 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10900 IOs, 4797308928 bytes 2026-02-22T21:48:41.175 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11000 IOs, 4852362240 bytes 2026-02-22T21:48:41.244 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11100 IOs, 4949245952 bytes 2026-02-22T21:48:41.291 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11200 IOs, 4976001024 bytes 2026-02-22T21:48:41.336 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11300 IOs, 5002948608 bytes 2026-02-22T21:48:41.391 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11400 IOs, 5069407232 bytes 2026-02-22T21:48:41.425 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11500 IOs, 5086147584 bytes 2026-02-22T21:48:41.477 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11600 IOs, 5140043776 bytes 2026-02-22T21:48:41.533 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11700 IOs, 5209741312 bytes 2026-02-22T21:48:41.573 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11800 IOs, 5238875136 bytes 2026-02-22T21:48:41.605 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11900 IOs, 5273216000 bytes 2026-02-22T21:48:41.651 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12000 IOs, 5288879104 bytes 2026-02-22T21:48:41.706 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12100 IOs, 5360765952 bytes 2026-02-22T21:48:41.763 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12200 IOs, 5425427456 bytes 2026-02-22T21:48:41.808 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12300 IOs, 5465465856 bytes 2026-02-22T21:48:41.844 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12400 IOs, 5487583232 bytes 2026-02-22T21:48:41.907 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12500 IOs, 5564485632 bytes 2026-02-22T21:48:41.961 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12600 IOs, 5635850240 bytes 2026-02-22T21:48:41.990 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12700 IOs, 5642791936 bytes 2026-02-22T21:48:42.019 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12800 IOs, 5652935680 bytes 2026-02-22T21:48:42.045 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12900 IOs, 5653539840 bytes 2026-02-22T21:48:42.091 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13000 IOs, 5705674752 bytes 2026-02-22T21:48:42.148 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13100 IOs, 5768426496 bytes 2026-02-22T21:48:42.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13200 IOs, 5788236800 bytes 2026-02-22T21:48:42.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13300 IOs, 5803587584 bytes 2026-02-22T21:48:42.261 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13400 IOs, 5858827264 bytes 2026-02-22T21:48:42.317 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13500 IOs, 5914904576 bytes 2026-02-22T21:48:42.370 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13600 IOs, 5960406016 bytes 2026-02-22T21:48:42.399 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13700 IOs, 5973550080 bytes 2026-02-22T21:48:42.458 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13800 IOs, 6033120256 bytes 2026-02-22T21:48:42.486 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13900 IOs, 6033745920 bytes 2026-02-22T21:48:42.513 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14000 IOs, 6034373632 bytes 2026-02-22T21:48:42.550 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14100 IOs, 6059199488 bytes 2026-02-22T21:48:42.602 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14200 IOs, 6118981632 bytes 2026-02-22T21:48:42.635 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14300 IOs, 6145789952 bytes 2026-02-22T21:48:42.675 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14400 IOs, 6163760128 bytes 2026-02-22T21:48:42.716 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14500 IOs, 6198388736 bytes 2026-02-22T21:48:42.769 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14600 IOs, 6258775040 bytes 2026-02-22T21:48:42.816 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14700 IOs, 6301983744 bytes 2026-02-22T21:48:42.869 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14800 IOs, 6366528512 bytes 2026-02-22T21:48:42.917 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14900 IOs, 6422378496 bytes 2026-02-22T21:48:42.960 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15000 IOs, 6457569280 bytes 2026-02-22T21:48:43.009 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15100 IOs, 6522284032 bytes 2026-02-22T21:48:43.056 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15200 IOs, 6547331072 bytes 2026-02-22T21:48:43.094 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15300 IOs, 6581592064 bytes 2026-02-22T21:48:43.124 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ error_code=124 2026-02-22T21:48:43.125 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set -e 2026-02-22T21:48:43.125 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' 124 -eq 124 ']' 2026-02-22T21:48:43.125 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:48:43.125 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T21:48:43.125 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_6 2026-02-22T21:48:43.125 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image_and_enable_mirror cluster2 mirror image_6 snapshot 128M 2026-02-22T21:48:43.125 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:48:43.125 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:48:43.125 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:48:43.126 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:48:43.126 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_6 2026-02-22T21:48:43.126 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:48:43.126 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T21:48:43.126 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n snapshot ']' 2026-02-22T21:48:43.126 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:48:43.126 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image cluster2 mirror image_6 128M 2026-02-22T21:48:43.126 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:48:43.126 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:48:43.126 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:48:43.126 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:48:43.126 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_6 2026-02-22T21:48:43.126 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:48:43.127 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local size=128 2026-02-22T21:48:43.127 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n 128M ']' 2026-02-22T21:48:43.127 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ size=128M 2026-02-22T21:48:43.127 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:48:43.127 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_6 2026-02-22T21:48:43.154 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T21:48:43.154 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:48:43.154 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' image = image ']' 2026-02-22T21:48:43.154 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:48:43.154 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ enable_mirror cluster2 mirror image_6 snapshot 2026-02-22T21:48:43.155 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:48:43.155 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:48:43.155 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_6 2026-02-22T21:48:43.155 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T21:48:43.155 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_6 snapshot 2026-02-22T21:48:44.133 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Mirroring enabled 2026-02-22T21:48:44.137 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 info mirror/image_6 2026-02-22T21:48:44.163 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_6': 2026-02-22T21:48:44.163 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 128 MiB in 32 objects 2026-02-22T21:48:44.163 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T21:48:44.163 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 1 2026-02-22T21:48:44.163 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 12ae211c121e 2026-02-22T21:48:44.163 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.12ae211c121e 2026-02-22T21:48:44.163 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T21:48:44.163 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock 2026-02-22T21:48:44.163 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T21:48:44.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T21:48:44.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:48:43 2026 2026-02-22T21:48:44.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:48:43 2026 2026-02-22T21:48:44.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:48:43 2026 2026-02-22T21:48:44.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T21:48:44.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T21:48:44.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: 763b8c73-d0f5-43b5-9453-e1f95dcf6809 2026-02-22T21:48:44.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: true 2026-02-22T21:48:44.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n '' ']' 2026-02-22T21:48:44.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ stress_write_image cluster2 mirror image_6 2026-02-22T21:48:44.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:48:44.166 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:48:44.166 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_6 2026-02-22T21:48:44.166 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-22T21:48:44.166 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local duration=8 2026-02-22T21:48:44.166 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set +e 2026-02-22T21:48:44.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ timeout 8s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_6 --debug-rbd=20 --debug-journaler=20 2026-02-22T21:48:44.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:100 IOs, 14915584 bytes 2026-02-22T21:48:44.280 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:200 IOs, 66800640 bytes 2026-02-22T21:48:44.346 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:300 IOs, 140404736 bytes 2026-02-22T21:48:44.384 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:400 IOs, 176616448 bytes 2026-02-22T21:48:44.418 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:500 IOs, 189225984 bytes 2026-02-22T21:48:44.464 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:600 IOs, 238889984 bytes 2026-02-22T21:48:44.514 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:700 IOs, 287834112 bytes 2026-02-22T21:48:44.538 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:800 IOs, 288475136 bytes 2026-02-22T21:48:44.598 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:900 IOs, 363169792 bytes 2026-02-22T21:48:44.655 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1000 IOs, 432739328 bytes 2026-02-22T21:48:44.690 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1100 IOs, 456626176 bytes 2026-02-22T21:48:44.726 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1200 IOs, 489741312 bytes 2026-02-22T21:48:44.794 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1300 IOs, 580641792 bytes 2026-02-22T21:48:44.871 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1400 IOs, 713853952 bytes 2026-02-22T21:48:44.918 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1500 IOs, 741820416 bytes 2026-02-22T21:48:44.960 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1600 IOs, 778537984 bytes 2026-02-22T21:48:45.002 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1700 IOs, 815431680 bytes 2026-02-22T21:48:45.050 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1800 IOs, 869492736 bytes 2026-02-22T21:48:45.082 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1900 IOs, 881219584 bytes 2026-02-22T21:48:45.118 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2000 IOs, 912685056 bytes 2026-02-22T21:48:45.146 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2100 IOs, 913992704 bytes 2026-02-22T21:48:45.183 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2200 IOs, 964766720 bytes 2026-02-22T21:48:45.238 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2300 IOs, 1004952576 bytes 2026-02-22T21:48:45.279 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2400 IOs, 1031188480 bytes 2026-02-22T21:48:45.309 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2500 IOs, 1040028672 bytes 2026-02-22T21:48:45.357 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2600 IOs, 1078887424 bytes 2026-02-22T21:48:45.403 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2700 IOs, 1119544320 bytes 2026-02-22T21:48:45.447 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2800 IOs, 1152491520 bytes 2026-02-22T21:48:45.491 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2900 IOs, 1203915776 bytes 2026-02-22T21:48:45.542 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3000 IOs, 1243112448 bytes 2026-02-22T21:48:45.585 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3100 IOs, 1303105536 bytes 2026-02-22T21:48:45.626 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3200 IOs, 1334209536 bytes 2026-02-22T21:48:45.662 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3300 IOs, 1361729536 bytes 2026-02-22T21:48:45.709 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3400 IOs, 1401886720 bytes 2026-02-22T21:48:45.740 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3500 IOs, 1425536000 bytes 2026-02-22T21:48:45.796 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3600 IOs, 1477977088 bytes 2026-02-22T21:48:45.842 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3700 IOs, 1523900416 bytes 2026-02-22T21:48:45.883 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3800 IOs, 1561311232 bytes 2026-02-22T21:48:45.930 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3900 IOs, 1609393152 bytes 2026-02-22T21:48:45.976 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4000 IOs, 1663828992 bytes 2026-02-22T21:48:46.020 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4100 IOs, 1711766528 bytes 2026-02-22T21:48:46.055 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4200 IOs, 1712392192 bytes 2026-02-22T21:48:46.106 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4300 IOs, 1772179456 bytes 2026-02-22T21:48:46.151 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4400 IOs, 1816685568 bytes 2026-02-22T21:48:46.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4500 IOs, 1843326976 bytes 2026-02-22T21:48:46.227 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4600 IOs, 1890745344 bytes 2026-02-22T21:48:46.280 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4700 IOs, 1921942528 bytes 2026-02-22T21:48:46.357 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4800 IOs, 2039751680 bytes 2026-02-22T21:48:46.394 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4900 IOs, 2058822656 bytes 2026-02-22T21:48:46.444 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5000 IOs, 2108544000 bytes 2026-02-22T21:48:46.495 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5100 IOs, 2164476928 bytes 2026-02-22T21:48:46.538 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5200 IOs, 2217837568 bytes 2026-02-22T21:48:46.596 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5300 IOs, 2305464320 bytes 2026-02-22T21:48:46.649 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5400 IOs, 2354595840 bytes 2026-02-22T21:48:46.690 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5500 IOs, 2394931200 bytes 2026-02-22T21:48:46.729 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5600 IOs, 2422466560 bytes 2026-02-22T21:48:46.772 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5700 IOs, 2473383936 bytes 2026-02-22T21:48:46.835 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5800 IOs, 2552723456 bytes 2026-02-22T21:48:46.870 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5900 IOs, 2579526656 bytes 2026-02-22T21:48:46.915 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6000 IOs, 2637333504 bytes 2026-02-22T21:48:46.954 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6100 IOs, 2646690816 bytes 2026-02-22T21:48:46.980 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6200 IOs, 2649612288 bytes 2026-02-22T21:48:47.031 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6300 IOs, 2722245632 bytes 2026-02-22T21:48:47.065 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6400 IOs, 2731054080 bytes 2026-02-22T21:48:47.101 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6500 IOs, 2749872128 bytes 2026-02-22T21:48:47.150 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6600 IOs, 2802771968 bytes 2026-02-22T21:48:47.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6700 IOs, 2863385600 bytes 2026-02-22T21:48:47.266 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6800 IOs, 2938845184 bytes 2026-02-22T21:48:47.303 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6900 IOs, 2956423168 bytes 2026-02-22T21:48:47.350 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7000 IOs, 3003191296 bytes 2026-02-22T21:48:47.394 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7100 IOs, 3048186880 bytes 2026-02-22T21:48:47.424 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7200 IOs, 3055747072 bytes 2026-02-22T21:48:47.450 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7300 IOs, 3058552832 bytes 2026-02-22T21:48:47.499 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7400 IOs, 3114901504 bytes 2026-02-22T21:48:47.559 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7500 IOs, 3181403136 bytes 2026-02-22T21:48:47.585 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7600 IOs, 3182043136 bytes 2026-02-22T21:48:47.624 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7700 IOs, 3220153344 bytes 2026-02-22T21:48:47.674 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7800 IOs, 3270641664 bytes 2026-02-22T21:48:47.717 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7900 IOs, 3313560576 bytes 2026-02-22T21:48:47.767 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8000 IOs, 3367612416 bytes 2026-02-22T21:48:47.808 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8100 IOs, 3402872832 bytes 2026-02-22T21:48:47.841 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8200 IOs, 3416898560 bytes 2026-02-22T21:48:47.878 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8300 IOs, 3436949504 bytes 2026-02-22T21:48:47.916 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8400 IOs, 3467537408 bytes 2026-02-22T21:48:47.958 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8500 IOs, 3506597888 bytes 2026-02-22T21:48:47.991 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8600 IOs, 3522572288 bytes 2026-02-22T21:48:48.040 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8700 IOs, 3581559808 bytes 2026-02-22T21:48:48.087 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8800 IOs, 3633241088 bytes 2026-02-22T21:48:48.145 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8900 IOs, 3705693184 bytes 2026-02-22T21:48:48.185 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9000 IOs, 3728781312 bytes 2026-02-22T21:48:48.242 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9100 IOs, 3815794688 bytes 2026-02-22T21:48:48.290 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9200 IOs, 3861058560 bytes 2026-02-22T21:48:48.347 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9300 IOs, 3934754816 bytes 2026-02-22T21:48:48.404 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9400 IOs, 3991037952 bytes 2026-02-22T21:48:48.446 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9500 IOs, 4034143232 bytes 2026-02-22T21:48:48.492 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9600 IOs, 4088165376 bytes 2026-02-22T21:48:48.524 INFO:tasks.ceph.cluster2.osd.2.trial026.stderr:2026-02-22T21:48:48.523+0000 7f32e29d2640 -1 reset not still connected to 0x55768be300d0 2026-02-22T21:48:48.533 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9700 IOs, 4110391296 bytes 2026-02-22T21:48:48.588 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9800 IOs, 4174184448 bytes 2026-02-22T21:48:48.629 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9900 IOs, 4203827200 bytes 2026-02-22T21:48:48.659 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10000 IOs, 4222952448 bytes 2026-02-22T21:48:48.701 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10100 IOs, 4249302016 bytes 2026-02-22T21:48:48.751 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10200 IOs, 4312997888 bytes 2026-02-22T21:48:48.801 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10300 IOs, 4369805312 bytes 2026-02-22T21:48:48.842 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10400 IOs, 4406684672 bytes 2026-02-22T21:48:48.898 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10500 IOs, 4483836928 bytes 2026-02-22T21:48:48.961 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10600 IOs, 4579828736 bytes 2026-02-22T21:48:48.994 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10700 IOs, 4580434944 bytes 2026-02-22T21:48:49.032 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10800 IOs, 4606167040 bytes 2026-02-22T21:48:49.080 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10900 IOs, 4659864576 bytes 2026-02-22T21:48:49.123 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11000 IOs, 4693689344 bytes 2026-02-22T21:48:49.166 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11100 IOs, 4747891712 bytes 2026-02-22T21:48:49.218 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11200 IOs, 4813794304 bytes 2026-02-22T21:48:49.257 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11300 IOs, 4823714816 bytes 2026-02-22T21:48:49.292 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11400 IOs, 4844995584 bytes 2026-02-22T21:48:49.329 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11500 IOs, 4870139904 bytes 2026-02-22T21:48:49.372 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11600 IOs, 4917074944 bytes 2026-02-22T21:48:49.438 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11700 IOs, 4984639488 bytes 2026-02-22T21:48:49.504 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11800 IOs, 5082770432 bytes 2026-02-22T21:48:49.551 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11900 IOs, 5130238976 bytes 2026-02-22T21:48:49.584 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12000 IOs, 5147633664 bytes 2026-02-22T21:48:49.610 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12100 IOs, 5151879168 bytes 2026-02-22T21:48:49.662 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12200 IOs, 5212972032 bytes 2026-02-22T21:48:49.691 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12300 IOs, 5223001088 bytes 2026-02-22T21:48:49.737 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12400 IOs, 5265458176 bytes 2026-02-22T21:48:49.798 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12500 IOs, 5348868096 bytes 2026-02-22T21:48:49.838 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12600 IOs, 5383572480 bytes 2026-02-22T21:48:49.883 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12700 IOs, 5430267904 bytes 2026-02-22T21:48:49.917 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12800 IOs, 5447712768 bytes 2026-02-22T21:48:49.967 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12900 IOs, 5494242304 bytes 2026-02-22T21:48:50.007 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13000 IOs, 5518041088 bytes 2026-02-22T21:48:50.047 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13100 IOs, 5556337664 bytes 2026-02-22T21:48:50.077 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13200 IOs, 5570623488 bytes 2026-02-22T21:48:50.104 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13300 IOs, 5571237888 bytes 2026-02-22T21:48:50.150 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13400 IOs, 5616362496 bytes 2026-02-22T21:48:50.185 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13500 IOs, 5637426176 bytes 2026-02-22T21:48:50.213 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13600 IOs, 5638167552 bytes 2026-02-22T21:48:50.268 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13700 IOs, 5715281920 bytes 2026-02-22T21:48:50.307 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13800 IOs, 5731549184 bytes 2026-02-22T21:48:50.341 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13900 IOs, 5746178048 bytes 2026-02-22T21:48:50.413 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14000 IOs, 5854287872 bytes 2026-02-22T21:48:50.452 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14100 IOs, 5878822912 bytes 2026-02-22T21:48:50.487 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14200 IOs, 5902862336 bytes 2026-02-22T21:48:50.535 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14300 IOs, 5956259840 bytes 2026-02-22T21:48:50.578 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14400 IOs, 6013736960 bytes 2026-02-22T21:48:50.667 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14500 IOs, 6166797312 bytes 2026-02-22T21:48:50.733 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14600 IOs, 6232455168 bytes 2026-02-22T21:48:50.771 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14700 IOs, 6267112448 bytes 2026-02-22T21:48:50.803 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14800 IOs, 6269467648 bytes 2026-02-22T21:48:50.871 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14900 IOs, 6355604480 bytes 2026-02-22T21:48:50.915 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15000 IOs, 6400607232 bytes 2026-02-22T21:48:50.966 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15100 IOs, 6443521024 bytes 2026-02-22T21:48:51.000 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15200 IOs, 6460337152 bytes 2026-02-22T21:48:51.036 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15300 IOs, 6482349056 bytes 2026-02-22T21:48:51.098 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15400 IOs, 6567230464 bytes 2026-02-22T21:48:51.148 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15500 IOs, 6641942528 bytes 2026-02-22T21:48:51.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15600 IOs, 6710318080 bytes 2026-02-22T21:48:51.254 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15700 IOs, 6753807360 bytes 2026-02-22T21:48:51.298 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15800 IOs, 6787474432 bytes 2026-02-22T21:48:51.372 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15900 IOs, 6896548864 bytes 2026-02-22T21:48:51.414 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16000 IOs, 6928237568 bytes 2026-02-22T21:48:51.457 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16100 IOs, 6963832832 bytes 2026-02-22T21:48:51.499 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16200 IOs, 6997204992 bytes 2026-02-22T21:48:51.561 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16300 IOs, 7080634368 bytes 2026-02-22T21:48:51.597 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16400 IOs, 7093874688 bytes 2026-02-22T21:48:51.653 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16500 IOs, 7162487808 bytes 2026-02-22T21:48:51.712 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16600 IOs, 7243557888 bytes 2026-02-22T21:48:51.749 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16700 IOs, 7265422336 bytes 2026-02-22T21:48:51.791 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16800 IOs, 7303691264 bytes 2026-02-22T21:48:51.827 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16900 IOs, 7324377088 bytes 2026-02-22T21:48:51.892 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17000 IOs, 7408126976 bytes 2026-02-22T21:48:51.916 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17100 IOs, 7408717824 bytes 2026-02-22T21:48:51.966 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17200 IOs, 7473043456 bytes 2026-02-22T21:48:52.030 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17300 IOs, 7544434688 bytes 2026-02-22T21:48:52.101 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17400 IOs, 7643340800 bytes 2026-02-22T21:48:52.139 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17500 IOs, 7659685888 bytes 2026-02-22T21:48:52.175 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ error_code=124 2026-02-22T21:48:52.175 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set -e 2026-02-22T21:48:52.175 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' 124 -eq 124 ']' 2026-02-22T21:48:52.175 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:48:52.175 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T21:48:52.175 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_7 2026-02-22T21:48:52.175 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image_and_enable_mirror cluster2 mirror image_7 snapshot 128M 2026-02-22T21:48:52.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:48:52.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:48:52.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:48:52.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:48:52.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_7 2026-02-22T21:48:52.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:48:52.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T21:48:52.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n snapshot ']' 2026-02-22T21:48:52.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:48:52.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image cluster2 mirror image_7 128M 2026-02-22T21:48:52.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:48:52.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:48:52.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:48:52.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:48:52.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_7 2026-02-22T21:48:52.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:48:52.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local size=128 2026-02-22T21:48:52.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n 128M ']' 2026-02-22T21:48:52.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ size=128M 2026-02-22T21:48:52.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:48:52.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_7 2026-02-22T21:48:52.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T21:48:52.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:48:52.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' image = image ']' 2026-02-22T21:48:52.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:48:52.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ enable_mirror cluster2 mirror image_7 snapshot 2026-02-22T21:48:52.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:48:52.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:48:52.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_7 2026-02-22T21:48:52.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T21:48:52.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_7 snapshot 2026-02-22T21:48:53.140 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Mirroring enabled 2026-02-22T21:48:53.145 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 info mirror/image_7 2026-02-22T21:48:53.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_7': 2026-02-22T21:48:53.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 128 MiB in 32 objects 2026-02-22T21:48:53.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T21:48:53.169 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 1 2026-02-22T21:48:53.169 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 12b8ef8b4a2 2026-02-22T21:48:53.169 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.12b8ef8b4a2 2026-02-22T21:48:53.169 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T21:48:53.169 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock 2026-02-22T21:48:53.169 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T21:48:53.169 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T21:48:53.169 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:48:52 2026 2026-02-22T21:48:53.169 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:48:52 2026 2026-02-22T21:48:53.169 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:48:52 2026 2026-02-22T21:48:53.170 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T21:48:53.170 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T21:48:53.170 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: a383f702-7739-4131-97b5-30656d4ff503 2026-02-22T21:48:53.170 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: true 2026-02-22T21:48:53.171 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n '' ']' 2026-02-22T21:48:53.171 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ stress_write_image cluster2 mirror image_7 2026-02-22T21:48:53.171 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:48:53.171 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:48:53.171 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_7 2026-02-22T21:48:53.171 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-22T21:48:53.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local duration=6 2026-02-22T21:48:53.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set +e 2026-02-22T21:48:53.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ timeout 6s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_7 --debug-rbd=20 --debug-journaler=20 2026-02-22T21:48:53.258 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:100 IOs, 66965504 bytes 2026-02-22T21:48:53.307 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:200 IOs, 107222016 bytes 2026-02-22T21:48:53.352 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:300 IOs, 143941632 bytes 2026-02-22T21:48:53.412 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:400 IOs, 210825216 bytes 2026-02-22T21:48:53.450 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:500 IOs, 231951360 bytes 2026-02-22T21:48:53.495 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:600 IOs, 276559872 bytes 2026-02-22T21:48:53.545 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:700 IOs, 324395008 bytes 2026-02-22T21:48:53.579 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:800 IOs, 347816960 bytes 2026-02-22T21:48:53.626 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:900 IOs, 409184256 bytes 2026-02-22T21:48:53.674 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1000 IOs, 434381824 bytes 2026-02-22T21:48:53.735 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1100 IOs, 518602752 bytes 2026-02-22T21:48:53.786 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1200 IOs, 559635456 bytes 2026-02-22T21:48:53.829 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1300 IOs, 601543680 bytes 2026-02-22T21:48:53.870 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1400 IOs, 646315008 bytes 2026-02-22T21:48:53.933 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1500 IOs, 707469312 bytes 2026-02-22T21:48:53.979 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1600 IOs, 740621312 bytes 2026-02-22T21:48:54.011 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1700 IOs, 767147008 bytes 2026-02-22T21:48:54.069 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1800 IOs, 831261696 bytes 2026-02-22T21:48:54.119 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1900 IOs, 880988160 bytes 2026-02-22T21:48:54.160 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2000 IOs, 914744320 bytes 2026-02-22T21:48:54.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2100 IOs, 968171520 bytes 2026-02-22T21:48:54.251 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2200 IOs, 995156992 bytes 2026-02-22T21:48:54.292 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2300 IOs, 1029360640 bytes 2026-02-22T21:48:54.345 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2400 IOs, 1087145984 bytes 2026-02-22T21:48:54.391 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2500 IOs, 1121236992 bytes 2026-02-22T21:48:54.439 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2600 IOs, 1173932032 bytes 2026-02-22T21:48:54.478 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2700 IOs, 1193632768 bytes 2026-02-22T21:48:54.514 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2800 IOs, 1212893184 bytes 2026-02-22T21:48:54.558 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2900 IOs, 1244932096 bytes 2026-02-22T21:48:54.589 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3000 IOs, 1265498112 bytes 2026-02-22T21:48:54.628 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3100 IOs, 1272798208 bytes 2026-02-22T21:48:54.690 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3200 IOs, 1349364736 bytes 2026-02-22T21:48:54.718 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3300 IOs, 1351589888 bytes 2026-02-22T21:48:54.748 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3400 IOs, 1359788032 bytes 2026-02-22T21:48:54.792 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3500 IOs, 1403931648 bytes 2026-02-22T21:48:54.832 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3600 IOs, 1434649600 bytes 2026-02-22T21:48:54.876 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3700 IOs, 1473076224 bytes 2026-02-22T21:48:54.938 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3800 IOs, 1551699968 bytes 2026-02-22T21:48:54.993 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3900 IOs, 1600317440 bytes 2026-02-22T21:48:55.041 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4000 IOs, 1648215040 bytes 2026-02-22T21:48:55.074 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4100 IOs, 1664420864 bytes 2026-02-22T21:48:55.125 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4200 IOs, 1711393792 bytes 2026-02-22T21:48:55.185 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4300 IOs, 1772013568 bytes 2026-02-22T21:48:55.226 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4400 IOs, 1804403712 bytes 2026-02-22T21:48:55.263 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4500 IOs, 1821609984 bytes 2026-02-22T21:48:55.291 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4600 IOs, 1834208256 bytes 2026-02-22T21:48:55.359 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4700 IOs, 1918466048 bytes 2026-02-22T21:48:55.407 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4800 IOs, 1940376576 bytes 2026-02-22T21:48:55.447 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4900 IOs, 1967642624 bytes 2026-02-22T21:48:55.476 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5000 IOs, 1979032576 bytes 2026-02-22T21:48:55.532 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5100 IOs, 2027720704 bytes 2026-02-22T21:48:55.576 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5200 IOs, 2070098944 bytes 2026-02-22T21:48:55.625 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5300 IOs, 2105311232 bytes 2026-02-22T21:48:55.666 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5400 IOs, 2145071104 bytes 2026-02-22T21:48:55.714 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5500 IOs, 2188846080 bytes 2026-02-22T21:48:55.745 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5600 IOs, 2193391616 bytes 2026-02-22T21:48:55.786 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5700 IOs, 2225271808 bytes 2026-02-22T21:48:55.820 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5800 IOs, 2246545408 bytes 2026-02-22T21:48:55.857 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5900 IOs, 2251291648 bytes 2026-02-22T21:48:55.895 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6000 IOs, 2290596864 bytes 2026-02-22T21:48:55.956 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6100 IOs, 2351261696 bytes 2026-02-22T21:48:55.995 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6200 IOs, 2376682496 bytes 2026-02-22T21:48:56.052 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6300 IOs, 2428321792 bytes 2026-02-22T21:48:56.085 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6400 IOs, 2441585664 bytes 2026-02-22T21:48:56.115 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6500 IOs, 2445605888 bytes 2026-02-22T21:48:56.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6600 IOs, 2499296256 bytes 2026-02-22T21:48:56.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6700 IOs, 2502181888 bytes 2026-02-22T21:48:56.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6800 IOs, 2564224000 bytes 2026-02-22T21:48:56.291 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6900 IOs, 2590592000 bytes 2026-02-22T21:48:56.368 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7000 IOs, 2691565568 bytes 2026-02-22T21:48:56.423 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7100 IOs, 2741672960 bytes 2026-02-22T21:48:56.464 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7200 IOs, 2770071552 bytes 2026-02-22T21:48:56.523 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7300 IOs, 2835387392 bytes 2026-02-22T21:48:56.560 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7400 IOs, 2853222400 bytes 2026-02-22T21:48:56.593 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7500 IOs, 2863912960 bytes 2026-02-22T21:48:56.645 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7600 IOs, 2920166400 bytes 2026-02-22T21:48:56.699 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7700 IOs, 2977428480 bytes 2026-02-22T21:48:56.774 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7800 IOs, 3085342720 bytes 2026-02-22T21:48:56.830 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7900 IOs, 3146878976 bytes 2026-02-22T21:48:56.867 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8000 IOs, 3173196800 bytes 2026-02-22T21:48:56.899 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8100 IOs, 3187968000 bytes 2026-02-22T21:48:56.940 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8200 IOs, 3213723648 bytes 2026-02-22T21:48:56.983 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8300 IOs, 3244116992 bytes 2026-02-22T21:48:57.016 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8400 IOs, 3259418624 bytes 2026-02-22T21:48:57.068 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8500 IOs, 3325150208 bytes 2026-02-22T21:48:57.114 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8600 IOs, 3344581632 bytes 2026-02-22T21:48:57.141 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8700 IOs, 3345195008 bytes 2026-02-22T21:48:57.184 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8800 IOs, 3374940160 bytes 2026-02-22T21:48:57.219 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8900 IOs, 3393458176 bytes 2026-02-22T21:48:57.257 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9000 IOs, 3415523328 bytes 2026-02-22T21:48:57.297 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9100 IOs, 3443881984 bytes 2026-02-22T21:48:57.351 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9200 IOs, 3487683584 bytes 2026-02-22T21:48:57.405 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9300 IOs, 3542248448 bytes 2026-02-22T21:48:57.473 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9400 IOs, 3625076736 bytes 2026-02-22T21:48:57.517 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9500 IOs, 3667559424 bytes 2026-02-22T21:48:57.549 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9600 IOs, 3689701376 bytes 2026-02-22T21:48:57.617 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9700 IOs, 3766808576 bytes 2026-02-22T21:48:57.668 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9800 IOs, 3820739584 bytes 2026-02-22T21:48:57.698 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9900 IOs, 3843579904 bytes 2026-02-22T21:48:57.747 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10000 IOs, 3867876352 bytes 2026-02-22T21:48:57.785 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10100 IOs, 3889784832 bytes 2026-02-22T21:48:57.836 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10200 IOs, 3949123584 bytes 2026-02-22T21:48:57.886 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10300 IOs, 3985334272 bytes 2026-02-22T21:48:57.951 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10400 IOs, 4075584512 bytes 2026-02-22T21:48:58.001 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10500 IOs, 4118928384 bytes 2026-02-22T21:48:58.052 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10600 IOs, 4172321792 bytes 2026-02-22T21:48:58.113 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10700 IOs, 4264761344 bytes 2026-02-22T21:48:58.163 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10800 IOs, 4294999040 bytes 2026-02-22T21:48:58.218 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10900 IOs, 4370524160 bytes 2026-02-22T21:48:58.281 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11000 IOs, 4425248768 bytes 2026-02-22T21:48:58.339 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11100 IOs, 4491509760 bytes 2026-02-22T21:48:58.410 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11200 IOs, 4577344512 bytes 2026-02-22T21:48:58.455 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11300 IOs, 4613494784 bytes 2026-02-22T21:48:58.498 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11400 IOs, 4650352640 bytes 2026-02-22T21:48:58.529 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11500 IOs, 4662169600 bytes 2026-02-22T21:48:58.572 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11600 IOs, 4702684160 bytes 2026-02-22T21:48:58.611 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11700 IOs, 4736391168 bytes 2026-02-22T21:48:58.662 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11800 IOs, 4784680960 bytes 2026-02-22T21:48:58.711 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11900 IOs, 4846221312 bytes 2026-02-22T21:48:58.757 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12000 IOs, 4865842176 bytes 2026-02-22T21:48:58.812 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12100 IOs, 4917644288 bytes 2026-02-22T21:48:58.856 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12200 IOs, 4948118528 bytes 2026-02-22T21:48:58.907 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12300 IOs, 5014483968 bytes 2026-02-22T21:48:58.943 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12400 IOs, 5029228544 bytes 2026-02-22T21:48:58.998 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12500 IOs, 5070726144 bytes 2026-02-22T21:48:59.068 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12600 IOs, 5166244864 bytes 2026-02-22T21:48:59.112 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12700 IOs, 5198939136 bytes 2026-02-22T21:48:59.144 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12800 IOs, 5205814272 bytes 2026-02-22T21:48:59.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ error_code=124 2026-02-22T21:48:59.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set -e 2026-02-22T21:48:59.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' 124 -eq 124 ']' 2026-02-22T21:48:59.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:48:59.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T21:48:59.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_8 2026-02-22T21:48:59.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image_and_enable_mirror cluster2 mirror image_8 snapshot 128M 2026-02-22T21:48:59.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:48:59.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:48:59.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:48:59.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:48:59.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_8 2026-02-22T21:48:59.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:48:59.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T21:48:59.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n snapshot ']' 2026-02-22T21:48:59.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:48:59.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image cluster2 mirror image_8 128M 2026-02-22T21:48:59.180 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:48:59.180 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:48:59.180 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:48:59.180 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:48:59.180 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_8 2026-02-22T21:48:59.180 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:48:59.180 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local size=128 2026-02-22T21:48:59.180 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n 128M ']' 2026-02-22T21:48:59.180 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ size=128M 2026-02-22T21:48:59.180 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:48:59.180 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_8 2026-02-22T21:48:59.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T21:48:59.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:48:59.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' image = image ']' 2026-02-22T21:48:59.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:48:59.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ enable_mirror cluster2 mirror image_8 snapshot 2026-02-22T21:48:59.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:48:59.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:48:59.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_8 2026-02-22T21:48:59.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T21:48:59.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_8 snapshot 2026-02-22T21:49:00.149 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Mirroring enabled 2026-02-22T21:49:00.154 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 info mirror/image_8 2026-02-22T21:49:00.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_8': 2026-02-22T21:49:00.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 128 MiB in 32 objects 2026-02-22T21:49:00.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T21:49:00.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 1 2026-02-22T21:49:00.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 12c2bc22fa86 2026-02-22T21:49:00.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.12c2bc22fa86 2026-02-22T21:49:00.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T21:49:00.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock 2026-02-22T21:49:00.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T21:49:00.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T21:49:00.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:48:59 2026 2026-02-22T21:49:00.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:48:59 2026 2026-02-22T21:49:00.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:48:59 2026 2026-02-22T21:49:00.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T21:49:00.180 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T21:49:00.180 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: 6ec8d83c-acbe-4670-87cf-a72aa84767b5 2026-02-22T21:49:00.180 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: true 2026-02-22T21:49:00.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n '' ']' 2026-02-22T21:49:00.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ stress_write_image cluster2 mirror image_8 2026-02-22T21:49:00.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:49:00.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:49:00.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_8 2026-02-22T21:49:00.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-22T21:49:00.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local duration=11 2026-02-22T21:49:00.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set +e 2026-02-22T21:49:00.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ timeout 11s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_8 --debug-rbd=20 --debug-journaler=20 2026-02-22T21:49:00.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:100 IOs, 50659328 bytes 2026-02-22T21:49:00.308 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:200 IOs, 83578880 bytes 2026-02-22T21:49:00.362 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:300 IOs, 129411072 bytes 2026-02-22T21:49:00.419 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:400 IOs, 207736832 bytes 2026-02-22T21:49:00.472 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:500 IOs, 243363840 bytes 2026-02-22T21:49:00.522 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:600 IOs, 290042880 bytes 2026-02-22T21:49:00.579 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:700 IOs, 348358656 bytes 2026-02-22T21:49:00.647 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:800 IOs, 442567680 bytes 2026-02-22T21:49:00.697 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:900 IOs, 485625856 bytes 2026-02-22T21:49:00.731 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1000 IOs, 499445760 bytes 2026-02-22T21:49:00.799 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1100 IOs, 591046656 bytes 2026-02-22T21:49:00.848 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1200 IOs, 636306432 bytes 2026-02-22T21:49:00.882 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1300 IOs, 636876800 bytes 2026-02-22T21:49:00.944 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1400 IOs, 711218176 bytes 2026-02-22T21:49:00.987 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1500 IOs, 740350976 bytes 2026-02-22T21:49:01.043 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1600 IOs, 820910080 bytes 2026-02-22T21:49:01.105 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1700 IOs, 882566144 bytes 2026-02-22T21:49:01.148 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1800 IOs, 920002560 bytes 2026-02-22T21:49:01.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1900 IOs, 989829120 bytes 2026-02-22T21:49:01.244 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2000 IOs, 1009837056 bytes 2026-02-22T21:49:01.291 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2100 IOs, 1059418112 bytes 2026-02-22T21:49:01.340 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2200 IOs, 1125444608 bytes 2026-02-22T21:49:01.443 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2300 IOs, 1290557440 bytes 2026-02-22T21:49:01.495 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2400 IOs, 1346703360 bytes 2026-02-22T21:49:01.537 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2500 IOs, 1399436288 bytes 2026-02-22T21:49:01.581 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2600 IOs, 1419223040 bytes 2026-02-22T21:49:01.607 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2700 IOs, 1419831296 bytes 2026-02-22T21:49:01.646 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2800 IOs, 1466954752 bytes 2026-02-22T21:49:01.694 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2900 IOs, 1487172608 bytes 2026-02-22T21:49:01.733 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3000 IOs, 1510254592 bytes 2026-02-22T21:49:01.769 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3100 IOs, 1530855424 bytes 2026-02-22T21:49:01.813 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3200 IOs, 1564936192 bytes 2026-02-22T21:49:01.864 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3300 IOs, 1604963328 bytes 2026-02-22T21:49:01.924 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3400 IOs, 1665906688 bytes 2026-02-22T21:49:01.973 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3500 IOs, 1710467072 bytes 2026-02-22T21:49:02.022 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3600 IOs, 1750856704 bytes 2026-02-22T21:49:02.061 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3700 IOs, 1774565376 bytes 2026-02-22T21:49:02.111 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3800 IOs, 1819398144 bytes 2026-02-22T21:49:02.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3900 IOs, 1904195584 bytes 2026-02-22T21:49:02.231 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4000 IOs, 1940800512 bytes 2026-02-22T21:49:02.274 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4100 IOs, 1977246720 bytes 2026-02-22T21:49:02.333 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4200 IOs, 2039729152 bytes 2026-02-22T21:49:02.388 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4300 IOs, 2134958080 bytes 2026-02-22T21:49:02.442 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4400 IOs, 2138124288 bytes 2026-02-22T21:49:02.492 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4500 IOs, 2183583744 bytes 2026-02-22T21:49:02.545 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4600 IOs, 2223509504 bytes 2026-02-22T21:49:02.593 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4700 IOs, 2267859968 bytes 2026-02-22T21:49:02.646 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4800 IOs, 2327222272 bytes 2026-02-22T21:49:02.695 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4900 IOs, 2367873024 bytes 2026-02-22T21:49:02.756 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5000 IOs, 2433472512 bytes 2026-02-22T21:49:02.818 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5100 IOs, 2499320832 bytes 2026-02-22T21:49:02.890 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5200 IOs, 2596374528 bytes 2026-02-22T21:49:02.918 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5300 IOs, 2596995072 bytes 2026-02-22T21:49:02.967 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5400 IOs, 2634763264 bytes 2026-02-22T21:49:03.016 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5500 IOs, 2679016448 bytes 2026-02-22T21:49:03.081 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5600 IOs, 2760355840 bytes 2026-02-22T21:49:03.162 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5700 IOs, 2847981568 bytes 2026-02-22T21:49:03.227 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5800 IOs, 2929594368 bytes 2026-02-22T21:49:03.273 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5900 IOs, 2968308736 bytes 2026-02-22T21:49:03.317 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6000 IOs, 3026578432 bytes 2026-02-22T21:49:03.387 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6100 IOs, 3088821248 bytes 2026-02-22T21:49:03.443 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6200 IOs, 3145755648 bytes 2026-02-22T21:49:03.482 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6300 IOs, 3166107648 bytes 2026-02-22T21:49:03.521 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6400 IOs, 3204213760 bytes 2026-02-22T21:49:03.560 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6500 IOs, 3218356224 bytes 2026-02-22T21:49:03.606 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6600 IOs, 3253293056 bytes 2026-02-22T21:49:03.663 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6700 IOs, 3330667520 bytes 2026-02-22T21:49:03.707 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6800 IOs, 3340939264 bytes 2026-02-22T21:49:03.738 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6900 IOs, 3345933312 bytes 2026-02-22T21:49:03.786 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7000 IOs, 3384656896 bytes 2026-02-22T21:49:03.838 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7100 IOs, 3428001792 bytes 2026-02-22T21:49:03.904 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7200 IOs, 3504898048 bytes 2026-02-22T21:49:03.944 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7300 IOs, 3531683840 bytes 2026-02-22T21:49:03.987 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7400 IOs, 3565954048 bytes 2026-02-22T21:49:04.031 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7500 IOs, 3599381504 bytes 2026-02-22T21:49:04.073 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7600 IOs, 3621894144 bytes 2026-02-22T21:49:04.144 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7700 IOs, 3708225536 bytes 2026-02-22T21:49:04.184 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7800 IOs, 3732215808 bytes 2026-02-22T21:49:04.250 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7900 IOs, 3810820096 bytes 2026-02-22T21:49:04.304 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8000 IOs, 3871639552 bytes 2026-02-22T21:49:04.378 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8100 IOs, 3977159680 bytes 2026-02-22T21:49:04.430 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8200 IOs, 4015851520 bytes 2026-02-22T21:49:04.481 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8300 IOs, 4058120192 bytes 2026-02-22T21:49:04.540 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8400 IOs, 4123935744 bytes 2026-02-22T21:49:04.582 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8500 IOs, 4157739008 bytes 2026-02-22T21:49:04.633 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8600 IOs, 4216783872 bytes 2026-02-22T21:49:04.674 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8700 IOs, 4238852096 bytes 2026-02-22T21:49:04.716 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8800 IOs, 4285417472 bytes 2026-02-22T21:49:04.771 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8900 IOs, 4338089984 bytes 2026-02-22T21:49:04.818 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9000 IOs, 4372534272 bytes 2026-02-22T21:49:04.870 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9100 IOs, 4422360064 bytes 2026-02-22T21:49:04.917 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9200 IOs, 4461345792 bytes 2026-02-22T21:49:04.969 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9300 IOs, 4511213568 bytes 2026-02-22T21:49:05.012 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9400 IOs, 4543804416 bytes 2026-02-22T21:49:05.059 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9500 IOs, 4587198464 bytes 2026-02-22T21:49:05.103 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9600 IOs, 4635681792 bytes 2026-02-22T21:49:05.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9700 IOs, 4707210240 bytes 2026-02-22T21:49:05.229 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9800 IOs, 4765701120 bytes 2026-02-22T21:49:05.285 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9900 IOs, 4815233024 bytes 2026-02-22T21:49:05.325 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10000 IOs, 4831765504 bytes 2026-02-22T21:49:05.378 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10100 IOs, 4881658880 bytes 2026-02-22T21:49:05.428 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10200 IOs, 4915214336 bytes 2026-02-22T21:49:05.465 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10300 IOs, 4933266432 bytes 2026-02-22T21:49:05.514 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10400 IOs, 4963625984 bytes 2026-02-22T21:49:05.574 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10500 IOs, 5027854336 bytes 2026-02-22T21:49:05.609 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10600 IOs, 5037679616 bytes 2026-02-22T21:49:05.655 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10700 IOs, 5087563776 bytes 2026-02-22T21:49:05.700 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10800 IOs, 5130337280 bytes 2026-02-22T21:49:05.739 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10900 IOs, 5136416768 bytes 2026-02-22T21:49:05.787 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11000 IOs, 5181349888 bytes 2026-02-22T21:49:05.831 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11100 IOs, 5218622464 bytes 2026-02-22T21:49:05.860 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11200 IOs, 5219239936 bytes 2026-02-22T21:49:05.903 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11300 IOs, 5249870848 bytes 2026-02-22T21:49:05.958 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11400 IOs, 5333337088 bytes 2026-02-22T21:49:06.054 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11500 IOs, 5435645952 bytes 2026-02-22T21:49:06.100 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11600 IOs, 5469616128 bytes 2026-02-22T21:49:06.136 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11700 IOs, 5482813440 bytes 2026-02-22T21:49:06.174 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11800 IOs, 5503640576 bytes 2026-02-22T21:49:06.224 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11900 IOs, 5552020480 bytes 2026-02-22T21:49:06.261 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12000 IOs, 5568809984 bytes 2026-02-22T21:49:06.310 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12100 IOs, 5609793536 bytes 2026-02-22T21:49:06.355 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12200 IOs, 5635079168 bytes 2026-02-22T21:49:06.400 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12300 IOs, 5673241600 bytes 2026-02-22T21:49:06.465 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12400 IOs, 5776756736 bytes 2026-02-22T21:49:06.524 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12500 IOs, 5810886656 bytes 2026-02-22T21:49:06.560 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12600 IOs, 5822791680 bytes 2026-02-22T21:49:06.630 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12700 IOs, 5918212096 bytes 2026-02-22T21:49:06.677 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12800 IOs, 5967923200 bytes 2026-02-22T21:49:06.721 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12900 IOs, 5990839296 bytes 2026-02-22T21:49:06.772 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13000 IOs, 6047364096 bytes 2026-02-22T21:49:06.840 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13100 IOs, 6121761792 bytes 2026-02-22T21:49:06.896 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13200 IOs, 6185774080 bytes 2026-02-22T21:49:06.942 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13300 IOs, 6229167104 bytes 2026-02-22T21:49:07.022 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13400 IOs, 6339574784 bytes 2026-02-22T21:49:07.079 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13500 IOs, 6384942080 bytes 2026-02-22T21:49:07.123 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13600 IOs, 6420189184 bytes 2026-02-22T21:49:07.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13700 IOs, 6466984960 bytes 2026-02-22T21:49:07.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13800 IOs, 6470199296 bytes 2026-02-22T21:49:07.239 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13900 IOs, 6485638144 bytes 2026-02-22T21:49:07.308 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14000 IOs, 6573560832 bytes 2026-02-22T21:49:07.357 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14100 IOs, 6616963072 bytes 2026-02-22T21:49:07.401 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14200 IOs, 6686542848 bytes 2026-02-22T21:49:07.496 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14300 IOs, 6787195904 bytes 2026-02-22T21:49:07.540 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14400 IOs, 6820441088 bytes 2026-02-22T21:49:07.593 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14500 IOs, 6876420096 bytes 2026-02-22T21:49:07.635 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14600 IOs, 6893547520 bytes 2026-02-22T21:49:07.674 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14700 IOs, 6914118656 bytes 2026-02-22T21:49:07.721 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14800 IOs, 6955963392 bytes 2026-02-22T21:49:07.764 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14900 IOs, 7015145472 bytes 2026-02-22T21:49:07.810 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15000 IOs, 7038898176 bytes 2026-02-22T21:49:07.843 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15100 IOs, 7052167168 bytes 2026-02-22T21:49:07.900 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15200 IOs, 7117978624 bytes 2026-02-22T21:49:07.945 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15300 IOs, 7159921664 bytes 2026-02-22T21:49:07.987 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15400 IOs, 7189762048 bytes 2026-02-22T21:49:08.046 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15500 IOs, 7264718848 bytes 2026-02-22T21:49:08.095 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15600 IOs, 7307676672 bytes 2026-02-22T21:49:08.131 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15700 IOs, 7331803136 bytes 2026-02-22T21:49:08.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15800 IOs, 7346522112 bytes 2026-02-22T21:49:08.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15900 IOs, 7413505024 bytes 2026-02-22T21:49:08.302 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16000 IOs, 7525860352 bytes 2026-02-22T21:49:08.334 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16100 IOs, 7536942080 bytes 2026-02-22T21:49:08.378 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16200 IOs, 7567268864 bytes 2026-02-22T21:49:08.415 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16300 IOs, 7580695552 bytes 2026-02-22T21:49:08.469 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16400 IOs, 7625570304 bytes 2026-02-22T21:49:08.512 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16500 IOs, 7656896512 bytes 2026-02-22T21:49:08.547 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16600 IOs, 7675249664 bytes 2026-02-22T21:49:08.614 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16700 IOs, 7752910848 bytes 2026-02-22T21:49:08.646 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16800 IOs, 7764337664 bytes 2026-02-22T21:49:08.690 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16900 IOs, 7792168960 bytes 2026-02-22T21:49:08.731 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17000 IOs, 7815661568 bytes 2026-02-22T21:49:08.784 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17100 IOs, 7870195712 bytes 2026-02-22T21:49:08.827 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17200 IOs, 7899521024 bytes 2026-02-22T21:49:08.868 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17300 IOs, 7930408960 bytes 2026-02-22T21:49:08.911 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17400 IOs, 7955645440 bytes 2026-02-22T21:49:08.965 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17500 IOs, 8007673856 bytes 2026-02-22T21:49:08.991 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17600 IOs, 8008290304 bytes 2026-02-22T21:49:09.039 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17700 IOs, 8056740864 bytes 2026-02-22T21:49:09.090 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17800 IOs, 8101911552 bytes 2026-02-22T21:49:09.146 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17900 IOs, 8165363712 bytes 2026-02-22T21:49:09.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18000 IOs, 8232078336 bytes 2026-02-22T21:49:09.261 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18100 IOs, 8283189248 bytes 2026-02-22T21:49:09.308 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18200 IOs, 8323480576 bytes 2026-02-22T21:49:09.365 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18300 IOs, 8378808320 bytes 2026-02-22T21:49:09.408 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18400 IOs, 8403469312 bytes 2026-02-22T21:49:09.457 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18500 IOs, 8452668416 bytes 2026-02-22T21:49:09.509 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18600 IOs, 8501442560 bytes 2026-02-22T21:49:09.567 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18700 IOs, 8554669056 bytes 2026-02-22T21:49:09.619 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18800 IOs, 8605633536 bytes 2026-02-22T21:49:09.659 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18900 IOs, 8630666240 bytes 2026-02-22T21:49:09.702 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19000 IOs, 8661114880 bytes 2026-02-22T21:49:09.786 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19100 IOs, 8792951808 bytes 2026-02-22T21:49:09.830 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19200 IOs, 8827753472 bytes 2026-02-22T21:49:09.891 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19300 IOs, 8883524608 bytes 2026-02-22T21:49:09.939 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19400 IOs, 8924781568 bytes 2026-02-22T21:49:09.978 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19500 IOs, 8955138048 bytes 2026-02-22T21:49:10.031 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19600 IOs, 9007001600 bytes 2026-02-22T21:49:10.077 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19700 IOs, 9041202176 bytes 2026-02-22T21:49:10.162 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19800 IOs, 9155682304 bytes 2026-02-22T21:49:10.203 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19900 IOs, 9179133952 bytes 2026-02-22T21:49:10.250 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20000 IOs, 9218618368 bytes 2026-02-22T21:49:10.294 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20100 IOs, 9253014528 bytes 2026-02-22T21:49:10.333 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20200 IOs, 9278135296 bytes 2026-02-22T21:49:10.378 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20300 IOs, 9316528128 bytes 2026-02-22T21:49:10.443 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20400 IOs, 9383015424 bytes 2026-02-22T21:49:10.483 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20500 IOs, 9401286656 bytes 2026-02-22T21:49:10.545 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20600 IOs, 9468425216 bytes 2026-02-22T21:49:10.587 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20700 IOs, 9504816128 bytes 2026-02-22T21:49:10.638 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20800 IOs, 9536965632 bytes 2026-02-22T21:49:10.677 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20900 IOs, 9571276800 bytes 2026-02-22T21:49:10.731 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21000 IOs, 9617489920 bytes 2026-02-22T21:49:10.772 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21100 IOs, 9643863040 bytes 2026-02-22T21:49:10.820 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21200 IOs, 9671058432 bytes 2026-02-22T21:49:10.875 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21300 IOs, 9726904320 bytes 2026-02-22T21:49:10.911 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21400 IOs, 9749763072 bytes 2026-02-22T21:49:10.964 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21500 IOs, 9803899904 bytes 2026-02-22T21:49:10.993 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21600 IOs, 9804508160 bytes 2026-02-22T21:49:11.035 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21700 IOs, 9842174976 bytes 2026-02-22T21:49:11.083 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21800 IOs, 9873534976 bytes 2026-02-22T21:49:11.127 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21900 IOs, 9928332288 bytes 2026-02-22T21:49:11.188 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ error_code=124 2026-02-22T21:49:11.188 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set -e 2026-02-22T21:49:11.188 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' 124 -eq 124 ']' 2026-02-22T21:49:11.188 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:49:11.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T21:49:11.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_9 2026-02-22T21:49:11.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image_and_enable_mirror cluster2 mirror image_9 snapshot 128M 2026-02-22T21:49:11.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:49:11.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:49:11.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:49:11.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:49:11.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_9 2026-02-22T21:49:11.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:49:11.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T21:49:11.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n snapshot ']' 2026-02-22T21:49:11.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:49:11.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image cluster2 mirror image_9 128M 2026-02-22T21:49:11.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:49:11.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:49:11.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:49:11.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:49:11.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_9 2026-02-22T21:49:11.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:49:11.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local size=128 2026-02-22T21:49:11.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n 128M ']' 2026-02-22T21:49:11.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ size=128M 2026-02-22T21:49:11.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:49:11.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_9 2026-02-22T21:49:11.219 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T21:49:11.219 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:49:11.219 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' image = image ']' 2026-02-22T21:49:11.220 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:49:11.220 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ enable_mirror cluster2 mirror image_9 snapshot 2026-02-22T21:49:11.220 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:49:11.220 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:49:11.220 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_9 2026-02-22T21:49:11.220 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T21:49:11.220 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_9 snapshot 2026-02-22T21:49:12.158 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Mirroring enabled 2026-02-22T21:49:12.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 info mirror/image_9 2026-02-22T21:49:12.188 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_9': 2026-02-22T21:49:12.188 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 128 MiB in 32 objects 2026-02-22T21:49:12.188 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T21:49:12.188 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 1 2026-02-22T21:49:12.188 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 12ca59aa1a07 2026-02-22T21:49:12.188 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.12ca59aa1a07 2026-02-22T21:49:12.188 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T21:49:12.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock 2026-02-22T21:49:12.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T21:49:12.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T21:49:12.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:49:11 2026 2026-02-22T21:49:12.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:49:11 2026 2026-02-22T21:49:12.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:49:11 2026 2026-02-22T21:49:12.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T21:49:12.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T21:49:12.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: cdba1f90-5b2a-43f5-aa74-a2544c65d250 2026-02-22T21:49:12.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: true 2026-02-22T21:49:12.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n '' ']' 2026-02-22T21:49:12.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ stress_write_image cluster2 mirror image_9 2026-02-22T21:49:12.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:49:12.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:49:12.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_9 2026-02-22T21:49:12.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-22T21:49:12.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local duration=7 2026-02-22T21:49:12.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set +e 2026-02-22T21:49:12.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ timeout 7s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_9 --debug-rbd=20 --debug-journaler=20 2026-02-22T21:49:12.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:100 IOs, 73403392 bytes 2026-02-22T21:49:12.332 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:200 IOs, 92777472 bytes 2026-02-22T21:49:12.373 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:300 IOs, 122986496 bytes 2026-02-22T21:49:12.409 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:400 IOs, 141258752 bytes 2026-02-22T21:49:12.446 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:500 IOs, 159549440 bytes 2026-02-22T21:49:12.519 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:600 IOs, 256484352 bytes 2026-02-22T21:49:12.562 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:700 IOs, 279448576 bytes 2026-02-22T21:49:12.597 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:800 IOs, 299274240 bytes 2026-02-22T21:49:12.625 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:900 IOs, 307009536 bytes 2026-02-22T21:49:12.654 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1000 IOs, 307619840 bytes 2026-02-22T21:49:12.717 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1100 IOs, 399827968 bytes 2026-02-22T21:49:12.759 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1200 IOs, 421082112 bytes 2026-02-22T21:49:12.784 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1300 IOs, 430838784 bytes 2026-02-22T21:49:12.847 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1400 IOs, 509434880 bytes 2026-02-22T21:49:12.888 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1500 IOs, 520924160 bytes 2026-02-22T21:49:12.931 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1600 IOs, 561796096 bytes 2026-02-22T21:49:12.968 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1700 IOs, 582908928 bytes 2026-02-22T21:49:13.015 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1800 IOs, 612276224 bytes 2026-02-22T21:49:13.062 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1900 IOs, 660182016 bytes 2026-02-22T21:49:13.115 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2000 IOs, 698058752 bytes 2026-02-22T21:49:13.159 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2100 IOs, 742975488 bytes 2026-02-22T21:49:13.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2200 IOs, 779196416 bytes 2026-02-22T21:49:13.249 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2300 IOs, 808999936 bytes 2026-02-22T21:49:13.302 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2400 IOs, 861326336 bytes 2026-02-22T21:49:13.351 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2500 IOs, 920579072 bytes 2026-02-22T21:49:13.435 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2600 IOs, 1019675648 bytes 2026-02-22T21:49:13.467 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2700 IOs, 1028384768 bytes 2026-02-22T21:49:13.508 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2800 IOs, 1049517056 bytes 2026-02-22T21:49:13.557 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2900 IOs, 1094714368 bytes 2026-02-22T21:49:13.590 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3000 IOs, 1110994944 bytes 2026-02-22T21:49:13.618 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3100 IOs, 1123044352 bytes 2026-02-22T21:49:13.660 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3200 IOs, 1137455104 bytes 2026-02-22T21:49:13.736 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3300 IOs, 1225277440 bytes 2026-02-22T21:49:13.780 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3400 IOs, 1262688256 bytes 2026-02-22T21:49:13.814 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3500 IOs, 1280119808 bytes 2026-02-22T21:49:13.857 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3600 IOs, 1310676992 bytes 2026-02-22T21:49:13.920 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3700 IOs, 1392293888 bytes 2026-02-22T21:49:13.953 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3800 IOs, 1428984832 bytes 2026-02-22T21:49:14.038 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3900 IOs, 1533490176 bytes 2026-02-22T21:49:14.073 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4000 IOs, 1566763008 bytes 2026-02-22T21:49:14.133 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4100 IOs, 1612430336 bytes 2026-02-22T21:49:14.170 INFO:tasks.ceph.cluster2.osd.2.trial026.stderr:2026-02-22T21:49:14.167+0000 7f32e29d2640 -1 reset not still connected to 0x55769e50a1a0 2026-02-22T21:49:14.170 INFO:tasks.ceph.cluster2.osd.2.trial026.stderr:2026-02-22T21:49:14.167+0000 7f32e29d2640 -1 reset not still connected to 0x5576a15a0680 2026-02-22T21:49:14.172 INFO:tasks.ceph.cluster2.osd.2.trial026.stderr:2026-02-22T21:49:14.169+0000 7f32e29d2640 -1 reset not still connected to 0x55769e50a1a0 2026-02-22T21:49:14.172 INFO:tasks.ceph.cluster2.osd.2.trial026.stderr:2026-02-22T21:49:14.170+0000 7f32e29d2640 -1 reset not still connected to 0x5576a15a0680 2026-02-22T21:49:14.184 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4200 IOs, 1661553664 bytes 2026-02-22T21:49:14.236 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4300 IOs, 1705590784 bytes 2026-02-22T21:49:14.280 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4400 IOs, 1750077440 bytes 2026-02-22T21:49:14.328 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4500 IOs, 1803084800 bytes 2026-02-22T21:49:14.380 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4600 IOs, 1857044480 bytes 2026-02-22T21:49:14.421 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4700 IOs, 1897200640 bytes 2026-02-22T21:49:14.462 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4800 IOs, 1897816064 bytes 2026-02-22T21:49:14.502 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4900 IOs, 1923538944 bytes 2026-02-22T21:49:14.557 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5000 IOs, 1988824064 bytes 2026-02-22T21:49:14.605 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5100 IOs, 2028017664 bytes 2026-02-22T21:49:14.687 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5200 IOs, 2128807936 bytes 2026-02-22T21:49:14.725 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5300 IOs, 2152646656 bytes 2026-02-22T21:49:14.774 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5400 IOs, 2227829760 bytes 2026-02-22T21:49:14.815 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5500 IOs, 2238102528 bytes 2026-02-22T21:49:14.852 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5600 IOs, 2261750784 bytes 2026-02-22T21:49:14.899 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5700 IOs, 2311941120 bytes 2026-02-22T21:49:14.963 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5800 IOs, 2394612736 bytes 2026-02-22T21:49:15.032 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5900 IOs, 2475529216 bytes 2026-02-22T21:49:15.073 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6000 IOs, 2498629632 bytes 2026-02-22T21:49:15.141 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6100 IOs, 2584851456 bytes 2026-02-22T21:49:15.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6200 IOs, 2671639552 bytes 2026-02-22T21:49:15.275 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6300 IOs, 2767529984 bytes 2026-02-22T21:49:15.323 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6400 IOs, 2796716032 bytes 2026-02-22T21:49:15.369 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6500 IOs, 2855482368 bytes 2026-02-22T21:49:15.425 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6600 IOs, 2909444096 bytes 2026-02-22T21:49:15.466 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6700 IOs, 2938722304 bytes 2026-02-22T21:49:15.509 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6800 IOs, 2977227776 bytes 2026-02-22T21:49:15.550 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6900 IOs, 3010287616 bytes 2026-02-22T21:49:15.607 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7000 IOs, 3058341888 bytes 2026-02-22T21:49:15.656 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7100 IOs, 3110259712 bytes 2026-02-22T21:49:15.694 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7200 IOs, 3140590592 bytes 2026-02-22T21:49:15.744 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7300 IOs, 3188870144 bytes 2026-02-22T21:49:15.776 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7400 IOs, 3210060800 bytes 2026-02-22T21:49:15.812 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7500 IOs, 3224418304 bytes 2026-02-22T21:49:15.858 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7600 IOs, 3258514432 bytes 2026-02-22T21:49:15.903 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7700 IOs, 3301726208 bytes 2026-02-22T21:49:15.934 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7800 IOs, 3311241216 bytes 2026-02-22T21:49:15.970 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7900 IOs, 3346073600 bytes 2026-02-22T21:49:16.011 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8000 IOs, 3361114112 bytes 2026-02-22T21:49:16.061 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8100 IOs, 3407372288 bytes 2026-02-22T21:49:16.129 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8200 IOs, 3486525440 bytes 2026-02-22T21:49:16.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8300 IOs, 3525432320 bytes 2026-02-22T21:49:16.227 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8400 IOs, 3573725184 bytes 2026-02-22T21:49:16.300 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8500 IOs, 3663199232 bytes 2026-02-22T21:49:16.354 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8600 IOs, 3713465344 bytes 2026-02-22T21:49:16.407 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8700 IOs, 3770888192 bytes 2026-02-22T21:49:16.464 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8800 IOs, 3826351104 bytes 2026-02-22T21:49:16.534 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8900 IOs, 3931239424 bytes 2026-02-22T21:49:16.590 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9000 IOs, 3984737280 bytes 2026-02-22T21:49:16.654 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9100 IOs, 4060667904 bytes 2026-02-22T21:49:16.729 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9200 IOs, 4162078720 bytes 2026-02-22T21:49:16.775 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9300 IOs, 4212675584 bytes 2026-02-22T21:49:16.837 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9400 IOs, 4277881856 bytes 2026-02-22T21:49:16.891 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9500 IOs, 4327943168 bytes 2026-02-22T21:49:16.935 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9600 IOs, 4372286464 bytes 2026-02-22T21:49:16.963 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9700 IOs, 4378711040 bytes 2026-02-22T21:49:17.027 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9800 IOs, 4482539520 bytes 2026-02-22T21:49:17.132 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9900 IOs, 4617475072 bytes 2026-02-22T21:49:17.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10000 IOs, 4635259904 bytes 2026-02-22T21:49:17.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10100 IOs, 4638604288 bytes 2026-02-22T21:49:17.245 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10200 IOs, 4674727936 bytes 2026-02-22T21:49:17.289 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10300 IOs, 4711215104 bytes 2026-02-22T21:49:17.340 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10400 IOs, 4763624448 bytes 2026-02-22T21:49:17.383 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10500 IOs, 4801414144 bytes 2026-02-22T21:49:17.453 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10600 IOs, 4881751040 bytes 2026-02-22T21:49:17.515 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10700 IOs, 4938161152 bytes 2026-02-22T21:49:17.575 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10800 IOs, 4997639168 bytes 2026-02-22T21:49:17.620 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10900 IOs, 5037399040 bytes 2026-02-22T21:49:17.654 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11000 IOs, 5050480640 bytes 2026-02-22T21:49:17.703 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11100 IOs, 5097421824 bytes 2026-02-22T21:49:17.762 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11200 IOs, 5167843328 bytes 2026-02-22T21:49:17.809 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11300 IOs, 5200244736 bytes 2026-02-22T21:49:17.851 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11400 IOs, 5228030976 bytes 2026-02-22T21:49:17.910 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11500 IOs, 5303035904 bytes 2026-02-22T21:49:17.969 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11600 IOs, 5363254272 bytes 2026-02-22T21:49:18.014 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11700 IOs, 5396729856 bytes 2026-02-22T21:49:18.066 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11800 IOs, 5454490624 bytes 2026-02-22T21:49:18.149 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11900 IOs, 5565448192 bytes 2026-02-22T21:49:18.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12000 IOs, 5603954688 bytes 2026-02-22T21:49:18.228 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12100 IOs, 5620206592 bytes 2026-02-22T21:49:18.273 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12200 IOs, 5657478144 bytes 2026-02-22T21:49:18.316 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12300 IOs, 5679257600 bytes 2026-02-22T21:49:18.372 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12400 IOs, 5737668608 bytes 2026-02-22T21:49:18.433 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12500 IOs, 5808068608 bytes 2026-02-22T21:49:18.472 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12600 IOs, 5827028992 bytes 2026-02-22T21:49:18.536 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12700 IOs, 5906487296 bytes 2026-02-22T21:49:18.578 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12800 IOs, 5933996032 bytes 2026-02-22T21:49:18.615 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12900 IOs, 5954057216 bytes 2026-02-22T21:49:18.649 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13000 IOs, 5979271168 bytes 2026-02-22T21:49:18.691 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13100 IOs, 6011289600 bytes 2026-02-22T21:49:18.725 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13200 IOs, 6026499072 bytes 2026-02-22T21:49:18.789 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13300 IOs, 6090379264 bytes 2026-02-22T21:49:18.835 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13400 IOs, 6120945664 bytes 2026-02-22T21:49:18.877 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13500 IOs, 6157473792 bytes 2026-02-22T21:49:18.916 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13600 IOs, 6208156672 bytes 2026-02-22T21:49:18.958 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13700 IOs, 6227220480 bytes 2026-02-22T21:49:18.983 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13800 IOs, 6227855360 bytes 2026-02-22T21:49:19.014 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13900 IOs, 6242572288 bytes 2026-02-22T21:49:19.050 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14000 IOs, 6255803392 bytes 2026-02-22T21:49:19.087 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14100 IOs, 6288543744 bytes 2026-02-22T21:49:19.137 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14200 IOs, 6321545216 bytes 2026-02-22T21:49:19.183 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14300 IOs, 6359446528 bytes 2026-02-22T21:49:19.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ error_code=124 2026-02-22T21:49:19.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set -e 2026-02-22T21:49:19.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' 124 -eq 124 ']' 2026-02-22T21:49:19.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:49:19.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T21:49:19.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_10 2026-02-22T21:49:19.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image_and_enable_mirror cluster2 mirror image_10 snapshot 128M 2026-02-22T21:49:19.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:49:19.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:49:19.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:49:19.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:49:19.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_10 2026-02-22T21:49:19.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:49:19.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T21:49:19.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n snapshot ']' 2026-02-22T21:49:19.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:49:19.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image cluster2 mirror image_10 128M 2026-02-22T21:49:19.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:49:19.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:49:19.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:49:19.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:49:19.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_10 2026-02-22T21:49:19.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:49:19.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local size=128 2026-02-22T21:49:19.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n 128M ']' 2026-02-22T21:49:19.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ size=128M 2026-02-22T21:49:19.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:49:19.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_10 2026-02-22T21:49:19.228 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T21:49:19.228 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:49:19.229 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' image = image ']' 2026-02-22T21:49:19.229 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:49:19.229 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ enable_mirror cluster2 mirror image_10 snapshot 2026-02-22T21:49:19.229 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:49:19.229 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:49:19.229 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_10 2026-02-22T21:49:19.229 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T21:49:19.229 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_10 snapshot 2026-02-22T21:49:20.094 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Mirroring enabled 2026-02-22T21:49:20.099 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 info mirror/image_10 2026-02-22T21:49:20.123 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_10': 2026-02-22T21:49:20.123 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 128 MiB in 32 objects 2026-02-22T21:49:20.123 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T21:49:20.124 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 1 2026-02-22T21:49:20.124 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 12d59f62444b 2026-02-22T21:49:20.124 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.12d59f62444b 2026-02-22T21:49:20.124 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T21:49:20.124 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock 2026-02-22T21:49:20.124 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T21:49:20.124 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T21:49:20.124 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:49:19 2026 2026-02-22T21:49:20.124 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:49:19 2026 2026-02-22T21:49:20.124 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:49:19 2026 2026-02-22T21:49:20.125 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T21:49:20.125 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T21:49:20.125 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: 0f526ee1-bae4-4f17-9197-c721da6ab0c9 2026-02-22T21:49:20.125 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: true 2026-02-22T21:49:20.126 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n '' ']' 2026-02-22T21:49:20.126 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ stress_write_image cluster2 mirror image_10 2026-02-22T21:49:20.126 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:49:20.126 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:49:20.126 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_10 2026-02-22T21:49:20.126 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-22T21:49:20.127 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local duration=12 2026-02-22T21:49:20.127 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set +e 2026-02-22T21:49:20.127 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ timeout 12s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_10 --debug-rbd=20 --debug-journaler=20 2026-02-22T21:49:20.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:100 IOs, 29798400 bytes 2026-02-22T21:49:20.235 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:200 IOs, 45543424 bytes 2026-02-22T21:49:20.269 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:300 IOs, 62789632 bytes 2026-02-22T21:49:20.354 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:400 IOs, 178774016 bytes 2026-02-22T21:49:20.394 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:500 IOs, 218311680 bytes 2026-02-22T21:49:20.443 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:600 IOs, 264678400 bytes 2026-02-22T21:49:20.493 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:700 IOs, 305461248 bytes 2026-02-22T21:49:20.540 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:800 IOs, 354319360 bytes 2026-02-22T21:49:20.595 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:900 IOs, 416699392 bytes 2026-02-22T21:49:20.637 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1000 IOs, 463449088 bytes 2026-02-22T21:49:20.685 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1100 IOs, 486957056 bytes 2026-02-22T21:49:20.728 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1200 IOs, 528454656 bytes 2026-02-22T21:49:20.767 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1300 IOs, 552046592 bytes 2026-02-22T21:49:20.834 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1400 IOs, 644276224 bytes 2026-02-22T21:49:20.867 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1500 IOs, 653296640 bytes 2026-02-22T21:49:20.907 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1600 IOs, 684370944 bytes 2026-02-22T21:49:20.942 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1700 IOs, 711301120 bytes 2026-02-22T21:49:20.981 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1800 IOs, 751470592 bytes 2026-02-22T21:49:21.032 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1900 IOs, 796227584 bytes 2026-02-22T21:49:21.074 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2000 IOs, 838257664 bytes 2026-02-22T21:49:21.129 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2100 IOs, 891388928 bytes 2026-02-22T21:49:21.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2200 IOs, 955803648 bytes 2026-02-22T21:49:21.247 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2300 IOs, 1025503232 bytes 2026-02-22T21:49:21.275 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2400 IOs, 1038988288 bytes 2026-02-22T21:49:21.330 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2500 IOs, 1090796544 bytes 2026-02-22T21:49:21.383 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2600 IOs, 1153334272 bytes 2026-02-22T21:49:21.425 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2700 IOs, 1194854400 bytes 2026-02-22T21:49:21.473 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2800 IOs, 1247560704 bytes 2026-02-22T21:49:21.525 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2900 IOs, 1278669824 bytes 2026-02-22T21:49:21.587 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3000 IOs, 1346503680 bytes 2026-02-22T21:49:21.631 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3100 IOs, 1401292800 bytes 2026-02-22T21:49:21.697 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3200 IOs, 1464034304 bytes 2026-02-22T21:49:21.738 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3300 IOs, 1513332736 bytes 2026-02-22T21:49:21.789 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3400 IOs, 1550504960 bytes 2026-02-22T21:49:21.849 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3500 IOs, 1618561024 bytes 2026-02-22T21:49:21.893 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3600 IOs, 1653101568 bytes 2026-02-22T21:49:21.926 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3700 IOs, 1672823808 bytes 2026-02-22T21:49:21.978 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3800 IOs, 1721407488 bytes 2026-02-22T21:49:22.020 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3900 IOs, 1751184384 bytes 2026-02-22T21:49:22.093 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4000 IOs, 1839680512 bytes 2026-02-22T21:49:22.133 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4100 IOs, 1867234304 bytes 2026-02-22T21:49:22.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4200 IOs, 1907508224 bytes 2026-02-22T21:49:22.233 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4300 IOs, 1986211840 bytes 2026-02-22T21:49:22.285 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4400 IOs, 2011032576 bytes 2026-02-22T21:49:22.320 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4500 IOs, 2044637184 bytes 2026-02-22T21:49:22.382 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4600 IOs, 2116012032 bytes 2026-02-22T21:49:22.459 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4700 IOs, 2223345664 bytes 2026-02-22T21:49:22.505 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4800 IOs, 2258710528 bytes 2026-02-22T21:49:22.548 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4900 IOs, 2286171136 bytes 2026-02-22T21:49:22.598 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5000 IOs, 2332528640 bytes 2026-02-22T21:49:22.656 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5100 IOs, 2410702848 bytes 2026-02-22T21:49:22.720 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5200 IOs, 2484415488 bytes 2026-02-22T21:49:22.756 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5300 IOs, 2512100352 bytes 2026-02-22T21:49:22.801 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5400 IOs, 2555208704 bytes 2026-02-22T21:49:22.854 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5500 IOs, 2612947968 bytes 2026-02-22T21:49:22.908 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5600 IOs, 2667284480 bytes 2026-02-22T21:49:22.934 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5700 IOs, 2669997056 bytes 2026-02-22T21:49:22.989 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5800 IOs, 2712695808 bytes 2026-02-22T21:49:23.017 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5900 IOs, 2740161536 bytes 2026-02-22T21:49:23.064 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6000 IOs, 2763680768 bytes 2026-02-22T21:49:23.112 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6100 IOs, 2807702528 bytes 2026-02-22T21:49:23.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6200 IOs, 2869419008 bytes 2026-02-22T21:49:23.230 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6300 IOs, 2949551104 bytes 2026-02-22T21:49:23.279 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6400 IOs, 2988283904 bytes 2026-02-22T21:49:23.323 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6500 IOs, 3011687424 bytes 2026-02-22T21:49:23.369 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6600 IOs, 3052943360 bytes 2026-02-22T21:49:23.408 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6700 IOs, 3083748352 bytes 2026-02-22T21:49:23.454 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6800 IOs, 3120655360 bytes 2026-02-22T21:49:23.483 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6900 IOs, 3133292544 bytes 2026-02-22T21:49:23.532 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7000 IOs, 3189618688 bytes 2026-02-22T21:49:23.587 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7100 IOs, 3241142272 bytes 2026-02-22T21:49:23.627 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7200 IOs, 3264575488 bytes 2026-02-22T21:49:23.679 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7300 IOs, 3337829376 bytes 2026-02-22T21:49:23.715 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7400 IOs, 3344542720 bytes 2026-02-22T21:49:23.770 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7500 IOs, 3395487744 bytes 2026-02-22T21:49:23.817 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7600 IOs, 3442264064 bytes 2026-02-22T21:49:23.850 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7700 IOs, 3460042752 bytes 2026-02-22T21:49:23.900 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7800 IOs, 3506154496 bytes 2026-02-22T21:49:23.932 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7900 IOs, 3517133824 bytes 2026-02-22T21:49:23.982 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8000 IOs, 3566034944 bytes 2026-02-22T21:49:24.042 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8100 IOs, 3614474240 bytes 2026-02-22T21:49:24.108 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8200 IOs, 3720880128 bytes 2026-02-22T21:49:24.162 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8300 IOs, 3784840192 bytes 2026-02-22T21:49:24.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8400 IOs, 3816161280 bytes 2026-02-22T21:49:24.270 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8500 IOs, 3872065536 bytes 2026-02-22T21:49:24.306 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8600 IOs, 3882465280 bytes 2026-02-22T21:49:24.360 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8700 IOs, 3950230528 bytes 2026-02-22T21:49:24.434 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8800 IOs, 4048917504 bytes 2026-02-22T21:49:24.502 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8900 IOs, 4146948096 bytes 2026-02-22T21:49:24.528 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9000 IOs, 4157011968 bytes 2026-02-22T21:49:24.584 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9100 IOs, 4226331648 bytes 2026-02-22T21:49:24.633 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9200 IOs, 4251440128 bytes 2026-02-22T21:49:24.664 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9300 IOs, 4260497408 bytes 2026-02-22T21:49:24.704 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9400 IOs, 4307422208 bytes 2026-02-22T21:49:24.747 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9500 IOs, 4319075328 bytes 2026-02-22T21:49:24.799 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9600 IOs, 4367067136 bytes 2026-02-22T21:49:24.850 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9700 IOs, 4429244416 bytes 2026-02-22T21:49:24.886 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9800 IOs, 4445374464 bytes 2026-02-22T21:49:24.917 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9900 IOs, 4452311040 bytes 2026-02-22T21:49:24.972 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10000 IOs, 4500030464 bytes 2026-02-22T21:49:25.226 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10100 IOs, 4553524224 bytes 2026-02-22T21:49:25.268 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10200 IOs, 4570831872 bytes 2026-02-22T21:49:25.322 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10300 IOs, 4653830144 bytes 2026-02-22T21:49:25.387 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10400 IOs, 4719259648 bytes 2026-02-22T21:49:25.424 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10500 IOs, 4759599104 bytes 2026-02-22T21:49:25.457 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10600 IOs, 4760980480 bytes 2026-02-22T21:49:25.503 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10700 IOs, 4808023040 bytes 2026-02-22T21:49:25.528 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10800 IOs, 4812106752 bytes 2026-02-22T21:49:25.579 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10900 IOs, 4869898240 bytes 2026-02-22T21:49:25.613 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11000 IOs, 4882137088 bytes 2026-02-22T21:49:25.641 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11100 IOs, 4886400000 bytes 2026-02-22T21:49:25.695 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11200 IOs, 4936193024 bytes 2026-02-22T21:49:25.724 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11300 IOs, 4942583808 bytes 2026-02-22T21:49:25.766 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11400 IOs, 4980976640 bytes 2026-02-22T21:49:25.804 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11500 IOs, 4999501824 bytes 2026-02-22T21:49:25.852 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11600 IOs, 5065136128 bytes 2026-02-22T21:49:25.903 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11700 IOs, 5101750272 bytes 2026-02-22T21:49:25.945 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11800 IOs, 5131672576 bytes 2026-02-22T21:49:25.995 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11900 IOs, 5176513536 bytes 2026-02-22T21:49:26.060 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12000 IOs, 5266130944 bytes 2026-02-22T21:49:26.091 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12100 IOs, 5279908864 bytes 2026-02-22T21:49:26.125 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12200 IOs, 5296961536 bytes 2026-02-22T21:49:26.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12300 IOs, 5363685376 bytes 2026-02-22T21:49:26.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12400 IOs, 5383377920 bytes 2026-02-22T21:49:26.269 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12500 IOs, 5428929536 bytes 2026-02-22T21:49:26.325 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12600 IOs, 5486794752 bytes 2026-02-22T21:49:26.377 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12700 IOs, 5528044544 bytes 2026-02-22T21:49:26.438 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12800 IOs, 5595915264 bytes 2026-02-22T21:49:26.493 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12900 IOs, 5658997760 bytes 2026-02-22T21:49:26.552 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13000 IOs, 5704524800 bytes 2026-02-22T21:49:26.583 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13100 IOs, 5734547456 bytes 2026-02-22T21:49:26.641 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13200 IOs, 5776752640 bytes 2026-02-22T21:49:26.683 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13300 IOs, 5802114048 bytes 2026-02-22T21:49:26.721 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13400 IOs, 5841364992 bytes 2026-02-22T21:49:26.804 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13500 IOs, 5937659904 bytes 2026-02-22T21:49:26.844 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13600 IOs, 5963984896 bytes 2026-02-22T21:49:26.922 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13700 IOs, 6070971392 bytes 2026-02-22T21:49:26.964 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13800 IOs, 6104762368 bytes 2026-02-22T21:49:27.009 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13900 IOs, 6132860928 bytes 2026-02-22T21:49:27.048 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14000 IOs, 6150315008 bytes 2026-02-22T21:49:27.096 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14100 IOs, 6198460416 bytes 2026-02-22T21:49:27.124 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14200 IOs, 6200824832 bytes 2026-02-22T21:49:27.166 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14300 IOs, 6243406848 bytes 2026-02-22T21:49:27.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14400 IOs, 6292801536 bytes 2026-02-22T21:49:27.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14500 IOs, 6340427776 bytes 2026-02-22T21:49:27.332 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14600 IOs, 6408879104 bytes 2026-02-22T21:49:27.366 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14700 IOs, 6435775488 bytes 2026-02-22T21:49:27.396 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14800 IOs, 6440683520 bytes 2026-02-22T21:49:27.441 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14900 IOs, 6493299712 bytes 2026-02-22T21:49:27.485 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15000 IOs, 6517715968 bytes 2026-02-22T21:49:27.545 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15100 IOs, 6593015808 bytes 2026-02-22T21:49:27.607 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15200 IOs, 6651398144 bytes 2026-02-22T21:49:27.657 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15300 IOs, 6699049984 bytes 2026-02-22T21:49:27.685 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15400 IOs, 6699666432 bytes 2026-02-22T21:49:27.736 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15500 IOs, 6750703616 bytes 2026-02-22T21:49:27.789 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15600 IOs, 6836743168 bytes 2026-02-22T21:49:27.860 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15700 IOs, 6903947264 bytes 2026-02-22T21:49:27.906 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15800 IOs, 6945856512 bytes 2026-02-22T21:49:27.934 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15900 IOs, 6951037952 bytes 2026-02-22T21:49:27.971 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16000 IOs, 6982014976 bytes 2026-02-22T21:49:28.029 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16100 IOs, 7050293248 bytes 2026-02-22T21:49:28.090 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16200 IOs, 7133392896 bytes 2026-02-22T21:49:28.145 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16300 IOs, 7195289600 bytes 2026-02-22T21:49:28.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16400 IOs, 7246477312 bytes 2026-02-22T21:49:28.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16500 IOs, 7346930688 bytes 2026-02-22T21:49:28.322 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16600 IOs, 7378770944 bytes 2026-02-22T21:49:28.379 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16700 IOs, 7432317952 bytes 2026-02-22T21:49:28.404 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16800 IOs, 7432930304 bytes 2026-02-22T21:49:28.446 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16900 IOs, 7461407744 bytes 2026-02-22T21:49:28.500 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17000 IOs, 7522237440 bytes 2026-02-22T21:49:28.545 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17100 IOs, 7561877504 bytes 2026-02-22T21:49:28.580 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17200 IOs, 7584384000 bytes 2026-02-22T21:49:28.624 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17300 IOs, 7617630208 bytes 2026-02-22T21:49:28.655 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17400 IOs, 7622380544 bytes 2026-02-22T21:49:28.694 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17500 IOs, 7652634624 bytes 2026-02-22T21:49:28.746 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17600 IOs, 7701020672 bytes 2026-02-22T21:49:28.798 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17700 IOs, 7754789888 bytes 2026-02-22T21:49:28.848 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17800 IOs, 7797211136 bytes 2026-02-22T21:49:28.901 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17900 IOs, 7856598016 bytes 2026-02-22T21:49:28.964 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18000 IOs, 7940993024 bytes 2026-02-22T21:49:29.000 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18100 IOs, 7963085824 bytes 2026-02-22T21:49:29.027 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18200 IOs, 7963673600 bytes 2026-02-22T21:49:29.084 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18300 IOs, 8030191616 bytes 2026-02-22T21:49:29.117 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18400 IOs, 8037728256 bytes 2026-02-22T21:49:29.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18500 IOs, 8087015424 bytes 2026-02-22T21:49:29.226 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18600 IOs, 8165954560 bytes 2026-02-22T21:49:29.282 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18700 IOs, 8229819392 bytes 2026-02-22T21:49:29.330 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18800 IOs, 8283854848 bytes 2026-02-22T21:49:29.391 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18900 IOs, 8356847616 bytes 2026-02-22T21:49:29.429 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19000 IOs, 8382496768 bytes 2026-02-22T21:49:29.487 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19100 IOs, 8458563584 bytes 2026-02-22T21:49:29.540 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19200 IOs, 8503262208 bytes 2026-02-22T21:49:29.597 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19300 IOs, 8589500416 bytes 2026-02-22T21:49:29.660 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19400 IOs, 8641780736 bytes 2026-02-22T21:49:29.713 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19500 IOs, 8728605696 bytes 2026-02-22T21:49:29.790 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19600 IOs, 8821200896 bytes 2026-02-22T21:49:29.868 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19700 IOs, 8902625280 bytes 2026-02-22T21:49:29.898 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19800 IOs, 8903222272 bytes 2026-02-22T21:49:29.939 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19900 IOs, 8935272448 bytes 2026-02-22T21:49:29.978 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20000 IOs, 8961072128 bytes 2026-02-22T21:49:30.028 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20100 IOs, 9000695808 bytes 2026-02-22T21:49:30.070 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20200 IOs, 9021899776 bytes 2026-02-22T21:49:30.131 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20300 IOs, 9102186496 bytes 2026-02-22T21:49:30.174 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20400 IOs, 9133302784 bytes 2026-02-22T21:49:30.228 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20500 IOs, 9185135616 bytes 2026-02-22T21:49:30.267 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20600 IOs, 9208343552 bytes 2026-02-22T21:49:30.312 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20700 IOs, 9239858176 bytes 2026-02-22T21:49:30.345 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20800 IOs, 9246717952 bytes 2026-02-22T21:49:30.400 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20900 IOs, 9308679168 bytes 2026-02-22T21:49:30.445 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21000 IOs, 9349299200 bytes 2026-02-22T21:49:30.505 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21100 IOs, 9411035136 bytes 2026-02-22T21:49:30.552 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21200 IOs, 9446067200 bytes 2026-02-22T21:49:30.603 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21300 IOs, 9510365184 bytes 2026-02-22T21:49:30.684 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21400 IOs, 9620513792 bytes 2026-02-22T21:49:30.755 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21500 IOs, 9704312832 bytes 2026-02-22T21:49:30.786 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21600 IOs, 9708290048 bytes 2026-02-22T21:49:30.830 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21700 IOs, 9751015424 bytes 2026-02-22T21:49:30.863 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21800 IOs, 9769634816 bytes 2026-02-22T21:49:30.912 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21900 IOs, 9810642944 bytes 2026-02-22T21:49:30.944 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22000 IOs, 9811261440 bytes 2026-02-22T21:49:31.010 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22100 IOs, 9898741760 bytes 2026-02-22T21:49:31.045 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22200 IOs, 9914675200 bytes 2026-02-22T21:49:31.108 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22300 IOs, 9971717120 bytes 2026-02-22T21:49:31.155 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22400 IOs, 9995809792 bytes 2026-02-22T21:49:31.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22500 IOs, 10009992192 bytes 2026-02-22T21:49:31.230 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22600 IOs, 10046261248 bytes 2026-02-22T21:49:31.277 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22700 IOs, 10065895424 bytes 2026-02-22T21:49:31.314 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22800 IOs, 10083623936 bytes 2026-02-22T21:49:31.363 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22900 IOs, 10125342720 bytes 2026-02-22T21:49:31.393 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23000 IOs, 10148552704 bytes 2026-02-22T21:49:31.456 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23100 IOs, 10215472128 bytes 2026-02-22T21:49:31.486 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23200 IOs, 10230105088 bytes 2026-02-22T21:49:31.538 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23300 IOs, 10278466560 bytes 2026-02-22T21:49:31.605 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23400 IOs, 10325650432 bytes 2026-02-22T21:49:31.636 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23500 IOs, 10340633600 bytes 2026-02-22T21:49:31.674 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23600 IOs, 10355739648 bytes 2026-02-22T21:49:31.724 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23700 IOs, 10406932480 bytes 2026-02-22T21:49:31.773 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23800 IOs, 10457152512 bytes 2026-02-22T21:49:31.800 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23900 IOs, 10457763840 bytes 2026-02-22T21:49:31.852 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24000 IOs, 10512558080 bytes 2026-02-22T21:49:31.894 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24100 IOs, 10543380480 bytes 2026-02-22T21:49:31.935 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24200 IOs, 10575386624 bytes 2026-02-22T21:49:31.966 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24300 IOs, 10583089152 bytes 2026-02-22T21:49:32.039 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24400 IOs, 10680210432 bytes 2026-02-22T21:49:32.068 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24500 IOs, 10683955200 bytes 2026-02-22T21:49:32.133 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ error_code=124 2026-02-22T21:49:32.133 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set -e 2026-02-22T21:49:32.133 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' 124 -eq 124 ']' 2026-02-22T21:49:32.134 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:49:32.134 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T21:49:32.134 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_11 2026-02-22T21:49:32.134 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image_and_enable_mirror cluster2 mirror image_11 snapshot 128M 2026-02-22T21:49:32.134 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:49:32.134 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:49:32.134 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:49:32.134 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:49:32.135 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_11 2026-02-22T21:49:32.135 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:49:32.135 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T21:49:32.135 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n snapshot ']' 2026-02-22T21:49:32.135 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:49:32.135 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image cluster2 mirror image_11 128M 2026-02-22T21:49:32.135 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:49:32.135 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:49:32.136 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:49:32.136 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:49:32.136 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_11 2026-02-22T21:49:32.136 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:49:32.136 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local size=128 2026-02-22T21:49:32.136 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n 128M ']' 2026-02-22T21:49:32.136 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ size=128M 2026-02-22T21:49:32.136 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:49:32.136 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_11 2026-02-22T21:49:32.163 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T21:49:32.163 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:49:32.163 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' image = image ']' 2026-02-22T21:49:32.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:49:32.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ enable_mirror cluster2 mirror image_11 snapshot 2026-02-22T21:49:32.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:49:32.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:49:32.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_11 2026-02-22T21:49:32.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T21:49:32.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_11 snapshot 2026-02-22T21:49:33.103 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Mirroring enabled 2026-02-22T21:49:33.109 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 info mirror/image_11 2026-02-22T21:49:33.137 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_11': 2026-02-22T21:49:33.138 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 128 MiB in 32 objects 2026-02-22T21:49:33.138 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T21:49:33.138 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 1 2026-02-22T21:49:33.138 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 12dfefb572e4 2026-02-22T21:49:33.138 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.12dfefb572e4 2026-02-22T21:49:33.138 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T21:49:33.138 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock 2026-02-22T21:49:33.138 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T21:49:33.138 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T21:49:33.139 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:49:32 2026 2026-02-22T21:49:33.139 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:49:32 2026 2026-02-22T21:49:33.139 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:49:32 2026 2026-02-22T21:49:33.139 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T21:49:33.139 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T21:49:33.139 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: 69c86d55-4653-4f0e-a6ff-bc5fe92ac722 2026-02-22T21:49:33.139 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: true 2026-02-22T21:49:33.140 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n '' ']' 2026-02-22T21:49:33.140 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ stress_write_image cluster2 mirror image_11 2026-02-22T21:49:33.140 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:49:33.140 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:49:33.141 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_11 2026-02-22T21:49:33.141 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-22T21:49:33.141 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local duration=11 2026-02-22T21:49:33.141 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set +e 2026-02-22T21:49:33.142 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ timeout 11s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_11 --debug-rbd=20 --debug-journaler=20 2026-02-22T21:49:33.203 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:100 IOs, 4777984 bytes 2026-02-22T21:49:33.243 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:200 IOs, 30628864 bytes 2026-02-22T21:49:33.289 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:300 IOs, 83475456 bytes 2026-02-22T21:49:33.333 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:400 IOs, 105779200 bytes 2026-02-22T21:49:33.381 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:500 IOs, 150398976 bytes 2026-02-22T21:49:33.420 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:600 IOs, 180818944 bytes 2026-02-22T21:49:33.480 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:700 IOs, 256958464 bytes 2026-02-22T21:49:33.513 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:800 IOs, 262551552 bytes 2026-02-22T21:49:33.593 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:900 IOs, 384892928 bytes 2026-02-22T21:49:33.631 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1000 IOs, 416058368 bytes 2026-02-22T21:49:33.677 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1100 IOs, 437455872 bytes 2026-02-22T21:49:33.744 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1200 IOs, 513640448 bytes 2026-02-22T21:49:33.780 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1300 IOs, 529498112 bytes 2026-02-22T21:49:34.020 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1400 IOs, 554712064 bytes 2026-02-22T21:49:34.072 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1500 IOs, 606073856 bytes 2026-02-22T21:49:34.122 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1600 IOs, 659889152 bytes 2026-02-22T21:49:34.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1700 IOs, 737247232 bytes 2026-02-22T21:49:34.227 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1800 IOs, 757412864 bytes 2026-02-22T21:49:34.277 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1900 IOs, 801284096 bytes 2026-02-22T21:49:34.321 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2000 IOs, 835201024 bytes 2026-02-22T21:49:34.360 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2100 IOs, 861730816 bytes 2026-02-22T21:49:34.408 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2200 IOs, 920936448 bytes 2026-02-22T21:49:34.458 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2300 IOs, 961598464 bytes 2026-02-22T21:49:34.488 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2400 IOs, 977143808 bytes 2026-02-22T21:49:34.514 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2500 IOs, 996065280 bytes 2026-02-22T21:49:34.571 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2600 IOs, 1079286784 bytes 2026-02-22T21:49:34.632 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2700 IOs, 1118482432 bytes 2026-02-22T21:49:34.706 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2800 IOs, 1216357376 bytes 2026-02-22T21:49:34.751 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2900 IOs, 1246814208 bytes 2026-02-22T21:49:34.794 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3000 IOs, 1298361344 bytes 2026-02-22T21:49:34.872 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3100 IOs, 1386283008 bytes 2026-02-22T21:49:34.939 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3200 IOs, 1475756032 bytes 2026-02-22T21:49:34.989 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3300 IOs, 1517999104 bytes 2026-02-22T21:49:35.042 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3400 IOs, 1574201344 bytes 2026-02-22T21:49:35.100 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3500 IOs, 1642619904 bytes 2026-02-22T21:49:35.143 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3600 IOs, 1667500032 bytes 2026-02-22T21:49:35.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3700 IOs, 1703112704 bytes 2026-02-22T21:49:35.243 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3800 IOs, 1733641216 bytes 2026-02-22T21:49:35.309 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3900 IOs, 1827333120 bytes 2026-02-22T21:49:35.350 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4000 IOs, 1860291584 bytes 2026-02-22T21:49:35.412 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4100 IOs, 1937789952 bytes 2026-02-22T21:49:35.471 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4200 IOs, 2016877568 bytes 2026-02-22T21:49:35.516 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4300 IOs, 2071685120 bytes 2026-02-22T21:49:35.578 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4400 IOs, 2118093824 bytes 2026-02-22T21:49:35.627 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4500 IOs, 2160029696 bytes 2026-02-22T21:49:35.681 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4600 IOs, 2197432320 bytes 2026-02-22T21:49:35.738 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4700 IOs, 2264386560 bytes 2026-02-22T21:49:35.801 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4800 IOs, 2339688448 bytes 2026-02-22T21:49:35.840 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4900 IOs, 2367894528 bytes 2026-02-22T21:49:35.896 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5000 IOs, 2438906880 bytes 2026-02-22T21:49:35.941 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5100 IOs, 2478613504 bytes 2026-02-22T21:49:35.981 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5200 IOs, 2500414464 bytes 2026-02-22T21:49:36.039 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5300 IOs, 2565630976 bytes 2026-02-22T21:49:36.110 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5400 IOs, 2647289856 bytes 2026-02-22T21:49:36.161 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5500 IOs, 2693126144 bytes 2026-02-22T21:49:36.186 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5600 IOs, 2695719936 bytes 2026-02-22T21:49:36.244 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5700 IOs, 2785568768 bytes 2026-02-22T21:49:36.320 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5800 IOs, 2870303744 bytes 2026-02-22T21:49:36.355 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5900 IOs, 2886868992 bytes 2026-02-22T21:49:36.418 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6000 IOs, 2962176000 bytes 2026-02-22T21:49:36.452 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6100 IOs, 2978449408 bytes 2026-02-22T21:49:36.491 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6200 IOs, 3017691136 bytes 2026-02-22T21:49:36.520 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6300 IOs, 3018298368 bytes 2026-02-22T21:49:36.565 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6400 IOs, 3059153920 bytes 2026-02-22T21:49:36.598 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6500 IOs, 3092632576 bytes 2026-02-22T21:49:36.651 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6600 IOs, 3122037760 bytes 2026-02-22T21:49:36.683 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6700 IOs, 3131978752 bytes 2026-02-22T21:49:36.725 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6800 IOs, 3155568640 bytes 2026-02-22T21:49:36.755 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6900 IOs, 3159482368 bytes 2026-02-22T21:49:36.808 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7000 IOs, 3225755648 bytes 2026-02-22T21:49:36.840 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7100 IOs, 3227947008 bytes 2026-02-22T21:49:36.884 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7200 IOs, 3271065600 bytes 2026-02-22T21:49:36.919 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7300 IOs, 3288091648 bytes 2026-02-22T21:49:36.951 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7400 IOs, 3299782656 bytes 2026-02-22T21:49:36.986 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7500 IOs, 3320725504 bytes 2026-02-22T21:49:37.043 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7600 IOs, 3383653376 bytes 2026-02-22T21:49:37.078 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7700 IOs, 3397695488 bytes 2026-02-22T21:49:37.123 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7800 IOs, 3430596608 bytes 2026-02-22T21:49:37.158 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7900 IOs, 3446462464 bytes 2026-02-22T21:49:37.218 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8000 IOs, 3518408704 bytes 2026-02-22T21:49:37.255 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8100 IOs, 3539915776 bytes 2026-02-22T21:49:37.306 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8200 IOs, 3585142784 bytes 2026-02-22T21:49:37.382 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8300 IOs, 3676774400 bytes 2026-02-22T21:49:37.425 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8400 IOs, 3710960640 bytes 2026-02-22T21:49:37.471 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8500 IOs, 3737797632 bytes 2026-02-22T21:49:37.503 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8600 IOs, 3747140608 bytes 2026-02-22T21:49:37.554 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8700 IOs, 3793550336 bytes 2026-02-22T21:49:37.607 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8800 IOs, 3836826624 bytes 2026-02-22T21:49:37.668 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8900 IOs, 3902154752 bytes 2026-02-22T21:49:37.699 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9000 IOs, 3908740096 bytes 2026-02-22T21:49:37.745 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9100 IOs, 3944481792 bytes 2026-02-22T21:49:37.794 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9200 IOs, 3990870016 bytes 2026-02-22T21:49:37.837 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9300 IOs, 4023632896 bytes 2026-02-22T21:49:37.876 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9400 IOs, 4049997824 bytes 2026-02-22T21:49:37.902 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9500 IOs, 4050599936 bytes 2026-02-22T21:49:37.944 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9600 IOs, 4100940800 bytes 2026-02-22T21:49:38.017 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9700 IOs, 4169335808 bytes 2026-02-22T21:49:38.054 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9800 IOs, 4187580416 bytes 2026-02-22T21:49:38.097 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9900 IOs, 4224739328 bytes 2026-02-22T21:49:38.130 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10000 IOs, 4262202368 bytes 2026-02-22T21:49:38.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10100 IOs, 4287675392 bytes 2026-02-22T21:49:38.226 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10200 IOs, 4332855296 bytes 2026-02-22T21:49:38.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10300 IOs, 4376631296 bytes 2026-02-22T21:49:38.302 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10400 IOs, 4377266176 bytes 2026-02-22T21:49:38.352 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10500 IOs, 4419624960 bytes 2026-02-22T21:49:38.412 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10600 IOs, 4495316992 bytes 2026-02-22T21:49:38.453 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10700 IOs, 4522075136 bytes 2026-02-22T21:49:38.507 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10800 IOs, 4582011904 bytes 2026-02-22T21:49:38.537 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10900 IOs, 4595937280 bytes 2026-02-22T21:49:38.591 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11000 IOs, 4650757120 bytes 2026-02-22T21:49:38.628 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11100 IOs, 4673688576 bytes 2026-02-22T21:49:38.699 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11200 IOs, 4772838400 bytes 2026-02-22T21:49:38.737 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11300 IOs, 4800331776 bytes 2026-02-22T21:49:38.766 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11400 IOs, 4805266432 bytes 2026-02-22T21:49:38.801 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11500 IOs, 4819473408 bytes 2026-02-22T21:49:38.834 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11600 IOs, 4831090688 bytes 2026-02-22T21:49:38.876 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11700 IOs, 4868184064 bytes 2026-02-22T21:49:38.914 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11800 IOs, 4895924224 bytes 2026-02-22T21:49:38.945 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11900 IOs, 4902789120 bytes 2026-02-22T21:49:38.989 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12000 IOs, 4934638592 bytes 2026-02-22T21:49:39.032 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12100 IOs, 4972338176 bytes 2026-02-22T21:49:39.062 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12200 IOs, 4972939264 bytes 2026-02-22T21:49:39.125 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12300 IOs, 5048633344 bytes 2026-02-22T21:49:39.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12400 IOs, 5109822464 bytes 2026-02-22T21:49:39.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12500 IOs, 5120513024 bytes 2026-02-22T21:49:39.254 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12600 IOs, 5148071936 bytes 2026-02-22T21:49:39.283 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12700 IOs, 5148700672 bytes 2026-02-22T21:49:39.310 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12800 IOs, 5149322240 bytes 2026-02-22T21:49:39.368 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12900 IOs, 5205685248 bytes 2026-02-22T21:49:39.408 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13000 IOs, 5228946432 bytes 2026-02-22T21:49:39.453 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13100 IOs, 5270663168 bytes 2026-02-22T21:49:39.480 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13200 IOs, 5271276544 bytes 2026-02-22T21:49:39.520 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13300 IOs, 5297197056 bytes 2026-02-22T21:49:39.559 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13400 IOs, 5312243712 bytes 2026-02-22T21:49:39.625 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13500 IOs, 5391280128 bytes 2026-02-22T21:49:39.675 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13600 IOs, 5439816704 bytes 2026-02-22T21:49:39.730 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13700 IOs, 5493220352 bytes 2026-02-22T21:49:39.762 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13800 IOs, 5528610816 bytes 2026-02-22T21:49:39.805 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13900 IOs, 5558029312 bytes 2026-02-22T21:49:39.901 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14000 IOs, 5685044224 bytes 2026-02-22T21:49:39.937 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14100 IOs, 5691522048 bytes 2026-02-22T21:49:39.980 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14200 IOs, 5724579840 bytes 2026-02-22T21:49:40.056 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14300 IOs, 5833484288 bytes 2026-02-22T21:49:40.096 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14400 IOs, 5871355904 bytes 2026-02-22T21:49:40.153 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14500 IOs, 5941900288 bytes 2026-02-22T21:49:40.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14600 IOs, 5999443968 bytes 2026-02-22T21:49:40.256 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14700 IOs, 6047748096 bytes 2026-02-22T21:49:40.301 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14800 IOs, 6114519040 bytes 2026-02-22T21:49:40.379 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14900 IOs, 6223679488 bytes 2026-02-22T21:49:40.444 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15000 IOs, 6294713344 bytes 2026-02-22T21:49:40.512 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15100 IOs, 6366534656 bytes 2026-02-22T21:49:40.550 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15200 IOs, 6385908736 bytes 2026-02-22T21:49:40.605 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15300 IOs, 6428609536 bytes 2026-02-22T21:49:40.635 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15400 IOs, 6432811008 bytes 2026-02-22T21:49:40.679 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15500 IOs, 6470650880 bytes 2026-02-22T21:49:40.722 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15600 IOs, 6507513856 bytes 2026-02-22T21:49:40.798 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15700 IOs, 6644306944 bytes 2026-02-22T21:49:40.853 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15800 IOs, 6674708480 bytes 2026-02-22T21:49:40.928 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15900 IOs, 6781156352 bytes 2026-02-22T21:49:40.981 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16000 IOs, 6836118528 bytes 2026-02-22T21:49:41.040 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16100 IOs, 6912025600 bytes 2026-02-22T21:49:41.078 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16200 IOs, 6956073984 bytes 2026-02-22T21:49:41.122 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16300 IOs, 6976612352 bytes 2026-02-22T21:49:41.159 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16400 IOs, 6992887808 bytes 2026-02-22T21:49:41.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16500 IOs, 7005746176 bytes 2026-02-22T21:49:41.252 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16600 IOs, 7073474560 bytes 2026-02-22T21:49:41.310 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16700 IOs, 7141821440 bytes 2026-02-22T21:49:41.346 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16800 IOs, 7158505472 bytes 2026-02-22T21:49:41.411 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16900 IOs, 7232712704 bytes 2026-02-22T21:49:41.449 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17000 IOs, 7251031040 bytes 2026-02-22T21:49:41.478 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17100 IOs, 7259401216 bytes 2026-02-22T21:49:41.520 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17200 IOs, 7294771200 bytes 2026-02-22T21:49:41.585 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17300 IOs, 7377923072 bytes 2026-02-22T21:49:41.618 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17400 IOs, 7390011392 bytes 2026-02-22T21:49:41.675 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17500 IOs, 7449967616 bytes 2026-02-22T21:49:41.737 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17600 IOs, 7521507328 bytes 2026-02-22T21:49:41.795 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17700 IOs, 7581323264 bytes 2026-02-22T21:49:41.830 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17800 IOs, 7595547648 bytes 2026-02-22T21:49:41.872 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17900 IOs, 7638044672 bytes 2026-02-22T21:49:41.925 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18000 IOs, 7692026880 bytes 2026-02-22T21:49:41.992 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18100 IOs, 7775427584 bytes 2026-02-22T21:49:42.036 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18200 IOs, 7805190144 bytes 2026-02-22T21:49:42.076 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18300 IOs, 7833105408 bytes 2026-02-22T21:49:42.111 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18400 IOs, 7851173888 bytes 2026-02-22T21:49:42.137 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18500 IOs, 7851767808 bytes 2026-02-22T21:49:42.180 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18600 IOs, 7883069440 bytes 2026-02-22T21:49:42.239 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18700 IOs, 7970155520 bytes 2026-02-22T21:49:42.307 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18800 IOs, 8046559232 bytes 2026-02-22T21:49:42.363 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18900 IOs, 8100446208 bytes 2026-02-22T21:49:42.411 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19000 IOs, 8135235584 bytes 2026-02-22T21:49:42.465 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19100 IOs, 8190484480 bytes 2026-02-22T21:49:42.521 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19200 IOs, 8253028352 bytes 2026-02-22T21:49:42.561 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19300 IOs, 8289321984 bytes 2026-02-22T21:49:42.593 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19400 IOs, 8295654400 bytes 2026-02-22T21:49:42.634 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19500 IOs, 8326082560 bytes 2026-02-22T21:49:42.668 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19600 IOs, 8344653824 bytes 2026-02-22T21:49:42.719 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19700 IOs, 8388925440 bytes 2026-02-22T21:49:42.749 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19800 IOs, 8399891456 bytes 2026-02-22T21:49:42.801 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19900 IOs, 8465129472 bytes 2026-02-22T21:49:42.830 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20000 IOs, 8473448448 bytes 2026-02-22T21:49:42.889 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20100 IOs, 8548067328 bytes 2026-02-22T21:49:42.934 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20200 IOs, 8589458432 bytes 2026-02-22T21:49:42.961 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20300 IOs, 8590069760 bytes 2026-02-22T21:49:43.015 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20400 IOs, 8660524032 bytes 2026-02-22T21:49:43.043 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20500 IOs, 8669779968 bytes 2026-02-22T21:49:43.088 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20600 IOs, 8701869056 bytes 2026-02-22T21:49:43.141 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20700 IOs, 8752727040 bytes 2026-02-22T21:49:43.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20800 IOs, 8836765696 bytes 2026-02-22T21:49:43.268 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20900 IOs, 8905346048 bytes 2026-02-22T21:49:43.308 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21000 IOs, 8932901888 bytes 2026-02-22T21:49:43.344 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21100 IOs, 8947252224 bytes 2026-02-22T21:49:43.406 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21200 IOs, 9018237952 bytes 2026-02-22T21:49:43.433 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21300 IOs, 9018827776 bytes 2026-02-22T21:49:43.465 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21400 IOs, 9034402816 bytes 2026-02-22T21:49:43.516 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21500 IOs, 9080875008 bytes 2026-02-22T21:49:43.551 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21600 IOs, 9097287680 bytes 2026-02-22T21:49:43.587 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21700 IOs, 9118395392 bytes 2026-02-22T21:49:43.658 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21800 IOs, 9209114624 bytes 2026-02-22T21:49:43.681 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.0 2026-02-22T21:49:43.682 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Sent signal 15 2026-02-22T21:49:43.682 INFO:tasks.rbd_mirror_thrash:waiting for 7.572733435752386 secs before reviving daemons 2026-02-22T21:49:43.682 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial026.stderr:2026-02-22T21:49:43.681+0000 7f243200c640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 (PID: 53328) UID: 1000 2026-02-22T21:49:43.684 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21900 IOs, 9209716736 bytes 2026-02-22T21:49:43.746 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22000 IOs, 9279446016 bytes 2026-02-22T21:49:43.786 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22100 IOs, 9311377408 bytes 2026-02-22T21:49:43.860 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22200 IOs, 9409731584 bytes 2026-02-22T21:49:43.886 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22300 IOs, 9410362368 bytes 2026-02-22T21:49:43.955 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22400 IOs, 9507783680 bytes 2026-02-22T21:49:44.010 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22500 IOs, 9564364800 bytes 2026-02-22T21:49:44.055 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22600 IOs, 9598388224 bytes 2026-02-22T21:49:44.124 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22700 IOs, 9693592576 bytes 2026-02-22T21:49:44.147 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ error_code=124 2026-02-22T21:49:44.147 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set -e 2026-02-22T21:49:44.148 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' 124 -eq 124 ']' 2026-02-22T21:49:44.148 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:49:44.148 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T21:49:44.148 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_12 2026-02-22T21:49:44.148 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image_and_enable_mirror cluster2 mirror image_12 snapshot 128M 2026-02-22T21:49:44.148 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:49:44.148 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:49:44.148 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:49:44.148 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:49:44.149 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_12 2026-02-22T21:49:44.149 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:49:44.149 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T21:49:44.149 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n snapshot ']' 2026-02-22T21:49:44.149 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:49:44.149 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image cluster2 mirror image_12 128M 2026-02-22T21:49:44.149 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:49:44.149 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:49:44.149 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:49:44.149 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:49:44.149 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_12 2026-02-22T21:49:44.150 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:49:44.150 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local size=128 2026-02-22T21:49:44.150 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n 128M ']' 2026-02-22T21:49:44.150 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ size=128M 2026-02-22T21:49:44.150 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:49:44.150 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_12 2026-02-22T21:49:44.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T21:49:44.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:49:44.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' image = image ']' 2026-02-22T21:49:44.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:49:44.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ enable_mirror cluster2 mirror image_12 snapshot 2026-02-22T21:49:44.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:49:44.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:49:44.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_12 2026-02-22T21:49:44.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T21:49:44.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_12 snapshot 2026-02-22T21:49:45.114 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Mirroring enabled 2026-02-22T21:49:45.119 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 info mirror/image_12 2026-02-22T21:49:45.144 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_12': 2026-02-22T21:49:45.144 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 128 MiB in 32 objects 2026-02-22T21:49:45.144 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T21:49:45.144 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 1 2026-02-22T21:49:45.144 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 12e9865d67b7 2026-02-22T21:49:45.145 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.12e9865d67b7 2026-02-22T21:49:45.145 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T21:49:45.145 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock 2026-02-22T21:49:45.145 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T21:49:45.145 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T21:49:45.145 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:49:44 2026 2026-02-22T21:49:45.145 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:49:44 2026 2026-02-22T21:49:45.145 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:49:44 2026 2026-02-22T21:49:45.145 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T21:49:45.145 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T21:49:45.145 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: 82008379-765b-45b9-b3f2-d57783e88ea0 2026-02-22T21:49:45.146 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: true 2026-02-22T21:49:45.146 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n '' ']' 2026-02-22T21:49:45.147 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ stress_write_image cluster2 mirror image_12 2026-02-22T21:49:45.147 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:49:45.147 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:49:45.147 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_12 2026-02-22T21:49:45.147 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-22T21:49:45.148 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local duration=14 2026-02-22T21:49:45.148 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set +e 2026-02-22T21:49:45.148 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ timeout 14s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_12 --debug-rbd=20 --debug-journaler=20 2026-02-22T21:49:45.214 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:100 IOs, 9513984 bytes 2026-02-22T21:49:45.255 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:200 IOs, 46797824 bytes 2026-02-22T21:49:45.283 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:300 IOs, 52197376 bytes 2026-02-22T21:49:45.343 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:400 IOs, 116182016 bytes 2026-02-22T21:49:45.387 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:500 IOs, 149634048 bytes 2026-02-22T21:49:45.421 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:600 IOs, 175246336 bytes 2026-02-22T21:49:45.494 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:700 IOs, 265001984 bytes 2026-02-22T21:49:45.553 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:800 IOs, 333637632 bytes 2026-02-22T21:49:45.589 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:900 IOs, 369094656 bytes 2026-02-22T21:49:45.648 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1000 IOs, 431015936 bytes 2026-02-22T21:49:45.687 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1100 IOs, 435290112 bytes 2026-02-22T21:49:45.755 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1200 IOs, 524186624 bytes 2026-02-22T21:49:45.796 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1300 IOs, 541179904 bytes 2026-02-22T21:49:45.841 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1400 IOs, 578672640 bytes 2026-02-22T21:49:45.902 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1500 IOs, 644905984 bytes 2026-02-22T21:49:45.952 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1600 IOs, 710477824 bytes 2026-02-22T21:49:45.992 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1700 IOs, 719910912 bytes 2026-02-22T21:49:46.042 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1800 IOs, 768004096 bytes 2026-02-22T21:49:46.081 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1900 IOs, 792960000 bytes 2026-02-22T21:49:46.109 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2000 IOs, 797177856 bytes 2026-02-22T21:49:46.142 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2100 IOs, 812863488 bytes 2026-02-22T21:49:46.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2200 IOs, 858080256 bytes 2026-02-22T21:49:46.254 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2300 IOs, 950994944 bytes 2026-02-22T21:49:46.306 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2400 IOs, 983224320 bytes 2026-02-22T21:49:46.370 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2500 IOs, 1063956480 bytes 2026-02-22T21:49:46.416 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2600 IOs, 1119395840 bytes 2026-02-22T21:49:46.483 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2700 IOs, 1190034432 bytes 2026-02-22T21:49:46.528 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2800 IOs, 1231134720 bytes 2026-02-22T21:49:46.583 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2900 IOs, 1294313472 bytes 2026-02-22T21:49:46.622 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3000 IOs, 1340688384 bytes 2026-02-22T21:49:46.689 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3100 IOs, 1403106304 bytes 2026-02-22T21:49:46.736 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3200 IOs, 1446782976 bytes 2026-02-22T21:49:46.770 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3300 IOs, 1457012736 bytes 2026-02-22T21:49:46.819 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3400 IOs, 1504265216 bytes 2026-02-22T21:49:46.884 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3500 IOs, 1584264192 bytes 2026-02-22T21:49:46.923 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3600 IOs, 1624360960 bytes 2026-02-22T21:49:46.959 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3700 IOs, 1634364416 bytes 2026-02-22T21:49:47.003 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3800 IOs, 1663674368 bytes 2026-02-22T21:49:47.034 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3900 IOs, 1676850176 bytes 2026-02-22T21:49:47.078 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4000 IOs, 1699723264 bytes 2026-02-22T21:49:47.107 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4100 IOs, 1700310016 bytes 2026-02-22T21:49:47.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4200 IOs, 1776700416 bytes 2026-02-22T21:49:47.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4300 IOs, 1814353920 bytes 2026-02-22T21:49:47.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4400 IOs, 1901302784 bytes 2026-02-22T21:49:47.319 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4500 IOs, 1939475456 bytes 2026-02-22T21:49:47.373 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4600 IOs, 1983877120 bytes 2026-02-22T21:49:47.420 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4700 IOs, 2024753152 bytes 2026-02-22T21:49:47.453 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4800 IOs, 2036065280 bytes 2026-02-22T21:49:47.482 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4900 IOs, 2036664320 bytes 2026-02-22T21:49:47.708 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5000 IOs, 2046049280 bytes 2026-02-22T21:49:47.779 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5100 IOs, 2143739904 bytes 2026-02-22T21:49:47.826 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5200 IOs, 2176403456 bytes 2026-02-22T21:49:47.880 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5300 IOs, 2238809088 bytes 2026-02-22T21:49:47.927 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5400 IOs, 2291390464 bytes 2026-02-22T21:49:47.961 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5500 IOs, 2311234560 bytes 2026-02-22T21:49:48.006 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5600 IOs, 2350948352 bytes 2026-02-22T21:49:48.045 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5700 IOs, 2375946240 bytes 2026-02-22T21:49:48.089 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5800 IOs, 2424657920 bytes 2026-02-22T21:49:48.140 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5900 IOs, 2460173312 bytes 2026-02-22T21:49:48.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6000 IOs, 2557764608 bytes 2026-02-22T21:49:48.261 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6100 IOs, 2611495936 bytes 2026-02-22T21:49:48.299 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6200 IOs, 2630727680 bytes 2026-02-22T21:49:48.360 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6300 IOs, 2713265152 bytes 2026-02-22T21:49:48.408 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6400 IOs, 2754265088 bytes 2026-02-22T21:49:48.461 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6500 IOs, 2805450752 bytes 2026-02-22T21:49:48.492 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6600 IOs, 2822007808 bytes 2026-02-22T21:49:48.528 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6700 IOs, 2840092672 bytes 2026-02-22T21:49:48.569 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6800 IOs, 2873876480 bytes 2026-02-22T21:49:48.604 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6900 IOs, 2897346560 bytes 2026-02-22T21:49:48.661 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7000 IOs, 2974014464 bytes 2026-02-22T21:49:48.735 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7100 IOs, 3074193408 bytes 2026-02-22T21:49:48.770 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7200 IOs, 3082950656 bytes 2026-02-22T21:49:48.807 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7300 IOs, 3098968064 bytes 2026-02-22T21:49:48.887 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7400 IOs, 3214721024 bytes 2026-02-22T21:49:48.931 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7500 IOs, 3252453376 bytes 2026-02-22T21:49:48.985 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7600 IOs, 3318910976 bytes 2026-02-22T21:49:49.032 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7700 IOs, 3353939968 bytes 2026-02-22T21:49:49.084 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7800 IOs, 3411437568 bytes 2026-02-22T21:49:49.115 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7900 IOs, 3413249024 bytes 2026-02-22T21:49:49.159 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8000 IOs, 3448301568 bytes 2026-02-22T21:49:49.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8100 IOs, 3473170432 bytes 2026-02-22T21:49:49.247 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8200 IOs, 3509592064 bytes 2026-02-22T21:49:49.301 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8300 IOs, 3562941440 bytes 2026-02-22T21:49:49.371 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8400 IOs, 3633813504 bytes 2026-02-22T21:49:49.448 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8500 IOs, 3746336768 bytes 2026-02-22T21:49:49.505 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8600 IOs, 3815712768 bytes 2026-02-22T21:49:49.551 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8700 IOs, 3865110528 bytes 2026-02-22T21:49:49.590 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8800 IOs, 3889186816 bytes 2026-02-22T21:49:49.662 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8900 IOs, 3996244992 bytes 2026-02-22T21:49:49.715 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9000 IOs, 4056665088 bytes 2026-02-22T21:49:49.762 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9100 IOs, 4085980160 bytes 2026-02-22T21:49:49.797 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9200 IOs, 4123048960 bytes 2026-02-22T21:49:49.836 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9300 IOs, 4136343552 bytes 2026-02-22T21:49:49.901 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9400 IOs, 4224585728 bytes 2026-02-22T21:49:49.954 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9500 IOs, 4283351040 bytes 2026-02-22T21:49:50.000 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9600 IOs, 4330996736 bytes 2026-02-22T21:49:50.039 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9700 IOs, 4331612160 bytes 2026-02-22T21:49:50.117 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9800 IOs, 4434112512 bytes 2026-02-22T21:49:50.171 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9900 IOs, 4489987072 bytes 2026-02-22T21:49:50.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10000 IOs, 4515179520 bytes 2026-02-22T21:49:50.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10100 IOs, 4551069696 bytes 2026-02-22T21:49:50.290 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10200 IOs, 4575838208 bytes 2026-02-22T21:49:50.329 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10300 IOs, 4612156416 bytes 2026-02-22T21:49:50.381 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10400 IOs, 4645154816 bytes 2026-02-22T21:49:50.418 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10500 IOs, 4662821888 bytes 2026-02-22T21:49:50.468 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10600 IOs, 4708638720 bytes 2026-02-22T21:49:50.541 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10700 IOs, 4808747008 bytes 2026-02-22T21:49:50.576 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10800 IOs, 4817625088 bytes 2026-02-22T21:49:50.611 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10900 IOs, 4833234944 bytes 2026-02-22T21:49:50.657 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11000 IOs, 4860887040 bytes 2026-02-22T21:49:50.698 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11100 IOs, 4895133696 bytes 2026-02-22T21:49:50.751 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11200 IOs, 4915398656 bytes 2026-02-22T21:49:50.792 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11300 IOs, 4938881024 bytes 2026-02-22T21:49:50.843 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11400 IOs, 5010945024 bytes 2026-02-22T21:49:50.888 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11500 IOs, 5024150528 bytes 2026-02-22T21:49:50.944 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11600 IOs, 5090892800 bytes 2026-02-22T21:49:50.992 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11700 IOs, 5124912128 bytes 2026-02-22T21:49:51.042 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11800 IOs, 5171333120 bytes 2026-02-22T21:49:51.090 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11900 IOs, 5226242048 bytes 2026-02-22T21:49:51.160 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12000 IOs, 5306009600 bytes 2026-02-22T21:49:51.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12100 IOs, 5384991744 bytes 2026-02-22T21:49:51.255 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.0 2026-02-22T21:49:51.255 INFO:teuthology.orchestra.run:waiting for 600 2026-02-22T21:49:51.256 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.0 2026-02-22T21:49:51.256 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Restarting daemon 2026-02-22T21:49:51.256 DEBUG:teuthology.orchestra.run.trial026:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 2026-02-22T21:49:51.259 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Started 2026-02-22T21:49:51.260 INFO:tasks.rbd_mirror_thrash:waiting for 76.52566884774892 secs before thrashing 2026-02-22T21:49:51.274 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12200 IOs, 5424860160 bytes 2026-02-22T21:49:51.317 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12300 IOs, 5448496128 bytes 2026-02-22T21:49:51.373 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12400 IOs, 5510758400 bytes 2026-02-22T21:49:51.413 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12500 IOs, 5527665664 bytes 2026-02-22T21:49:51.456 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12600 IOs, 5550092288 bytes 2026-02-22T21:49:51.493 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12700 IOs, 5574211584 bytes 2026-02-22T21:49:51.528 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12800 IOs, 5584801792 bytes 2026-02-22T21:49:51.561 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12900 IOs, 5592712192 bytes 2026-02-22T21:49:51.605 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13000 IOs, 5633739776 bytes 2026-02-22T21:49:51.636 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13100 IOs, 5644064768 bytes 2026-02-22T21:49:51.665 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13200 IOs, 5644675072 bytes 2026-02-22T21:49:51.735 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13300 IOs, 5729498112 bytes 2026-02-22T21:49:51.786 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13400 IOs, 5772408832 bytes 2026-02-22T21:49:51.839 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13500 IOs, 5817089024 bytes 2026-02-22T21:49:51.889 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13600 IOs, 5858910208 bytes 2026-02-22T21:49:51.917 INFO:tasks.ceph.cluster2.osd.2.trial026.stderr:2026-02-22T21:49:51.915+0000 7f32e29d2640 -1 reset not still connected to 0x5576a15a0680 2026-02-22T21:49:51.943 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13700 IOs, 5912791040 bytes 2026-02-22T21:49:51.980 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13800 IOs, 5927682048 bytes 2026-02-22T21:49:52.016 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13900 IOs, 5944264704 bytes 2026-02-22T21:49:52.044 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14000 IOs, 5966281728 bytes 2026-02-22T21:49:52.099 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14100 IOs, 6002561024 bytes 2026-02-22T21:49:52.169 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14200 IOs, 6087791616 bytes 2026-02-22T21:49:52.220 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14300 IOs, 6138906624 bytes 2026-02-22T21:49:52.259 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14400 IOs, 6164753408 bytes 2026-02-22T21:49:52.306 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14500 IOs, 6198262784 bytes 2026-02-22T21:49:52.384 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14600 IOs, 6297943040 bytes 2026-02-22T21:49:52.440 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14700 IOs, 6371152896 bytes 2026-02-22T21:49:52.489 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14800 IOs, 6408621056 bytes 2026-02-22T21:49:52.529 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14900 IOs, 6436045824 bytes 2026-02-22T21:49:52.569 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15000 IOs, 6459254784 bytes 2026-02-22T21:49:52.625 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15100 IOs, 6516982784 bytes 2026-02-22T21:49:52.701 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15200 IOs, 6628320256 bytes 2026-02-22T21:49:52.758 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15300 IOs, 6692548608 bytes 2026-02-22T21:49:52.814 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15400 IOs, 6773168128 bytes 2026-02-22T21:49:52.873 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15500 IOs, 6843799552 bytes 2026-02-22T21:49:52.919 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15600 IOs, 6898393088 bytes 2026-02-22T21:49:52.962 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15700 IOs, 6930012160 bytes 2026-02-22T21:49:53.005 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15800 IOs, 6958221312 bytes 2026-02-22T21:49:53.052 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15900 IOs, 7001043968 bytes 2026-02-22T21:49:53.097 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16000 IOs, 7045436416 bytes 2026-02-22T21:49:53.142 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16100 IOs, 7085991936 bytes 2026-02-22T21:49:53.180 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16200 IOs, 7106188288 bytes 2026-02-22T21:49:53.252 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16300 IOs, 7192775680 bytes 2026-02-22T21:49:53.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16400 IOs, 7193660416 bytes 2026-02-22T21:49:53.315 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16500 IOs, 7203911680 bytes 2026-02-22T21:49:53.358 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16600 IOs, 7232084992 bytes 2026-02-22T21:49:53.412 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16700 IOs, 7286222848 bytes 2026-02-22T21:49:53.465 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16800 IOs, 7339252736 bytes 2026-02-22T21:49:53.508 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16900 IOs, 7389798400 bytes 2026-02-22T21:49:53.585 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17000 IOs, 7474964480 bytes 2026-02-22T21:49:53.641 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17100 IOs, 7536785408 bytes 2026-02-22T21:49:53.677 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17200 IOs, 7546781696 bytes 2026-02-22T21:49:53.738 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17300 IOs, 7622061056 bytes 2026-02-22T21:49:53.791 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17400 IOs, 7668301824 bytes 2026-02-22T21:49:53.849 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17500 IOs, 7731534848 bytes 2026-02-22T21:49:53.903 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17600 IOs, 7788933120 bytes 2026-02-22T21:49:53.949 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17700 IOs, 7830843392 bytes 2026-02-22T21:49:53.978 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17800 IOs, 7831452672 bytes 2026-02-22T21:49:54.025 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17900 IOs, 7868777472 bytes 2026-02-22T21:49:54.069 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18000 IOs, 7903732736 bytes 2026-02-22T21:49:54.112 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18100 IOs, 7941052416 bytes 2026-02-22T21:49:54.160 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18200 IOs, 7971468288 bytes 2026-02-22T21:49:54.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18300 IOs, 8045126656 bytes 2026-02-22T21:49:54.263 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18400 IOs, 8084200448 bytes 2026-02-22T21:49:54.296 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18500 IOs, 8095776768 bytes 2026-02-22T21:49:54.527 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18600 IOs, 8104747008 bytes 2026-02-22T21:49:54.559 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18700 IOs, 8121922560 bytes 2026-02-22T21:49:54.623 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18800 IOs, 8197397504 bytes 2026-02-22T21:49:54.671 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18900 IOs, 8248410112 bytes 2026-02-22T21:49:54.724 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19000 IOs, 8301336576 bytes 2026-02-22T21:49:54.758 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19100 IOs, 8321874944 bytes 2026-02-22T21:49:54.803 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19200 IOs, 8364109824 bytes 2026-02-22T21:49:54.834 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19300 IOs, 8372652032 bytes 2026-02-22T21:49:54.875 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19400 IOs, 8402660352 bytes 2026-02-22T21:49:54.922 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19500 IOs, 8464808960 bytes 2026-02-22T21:49:54.988 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19600 IOs, 8537850880 bytes 2026-02-22T21:49:55.035 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19700 IOs, 8574563328 bytes 2026-02-22T21:49:55.099 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19800 IOs, 8663778304 bytes 2026-02-22T21:49:55.149 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19900 IOs, 8720459776 bytes 2026-02-22T21:49:55.184 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20000 IOs, 8733487104 bytes 2026-02-22T21:49:55.256 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20100 IOs, 8829452288 bytes 2026-02-22T21:49:55.322 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20200 IOs, 8908011520 bytes 2026-02-22T21:49:55.361 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20300 IOs, 8928263168 bytes 2026-02-22T21:49:55.409 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20400 IOs, 8970536960 bytes 2026-02-22T21:49:55.455 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20500 IOs, 9013417984 bytes 2026-02-22T21:49:55.511 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20600 IOs, 9097247744 bytes 2026-02-22T21:49:55.574 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20700 IOs, 9148719104 bytes 2026-02-22T21:49:55.604 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20800 IOs, 9154596864 bytes 2026-02-22T21:49:55.658 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20900 IOs, 9206692864 bytes 2026-02-22T21:49:55.682 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21000 IOs, 9211770880 bytes 2026-02-22T21:49:55.728 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21100 IOs, 9245809664 bytes 2026-02-22T21:49:55.774 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21200 IOs, 9276231680 bytes 2026-02-22T21:49:55.826 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21300 IOs, 9330662400 bytes 2026-02-22T21:49:55.870 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21400 IOs, 9360128000 bytes 2026-02-22T21:49:55.925 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21500 IOs, 9421560832 bytes 2026-02-22T21:49:55.966 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21600 IOs, 9454372864 bytes 2026-02-22T21:49:56.004 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21700 IOs, 9474347008 bytes 2026-02-22T21:49:56.052 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21800 IOs, 9512383488 bytes 2026-02-22T21:49:56.105 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21900 IOs, 9566632960 bytes 2026-02-22T21:49:56.158 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22000 IOs, 9621497856 bytes 2026-02-22T21:49:56.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22100 IOs, 9655330816 bytes 2026-02-22T21:49:56.245 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22200 IOs, 9689538560 bytes 2026-02-22T21:49:56.286 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22300 IOs, 9715133440 bytes 2026-02-22T21:49:56.324 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22400 IOs, 9751646208 bytes 2026-02-22T21:49:56.392 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22500 IOs, 9819716608 bytes 2026-02-22T21:49:56.424 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22600 IOs, 9835134976 bytes 2026-02-22T21:49:56.476 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22700 IOs, 9881477120 bytes 2026-02-22T21:49:56.525 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22800 IOs, 9927623680 bytes 2026-02-22T21:49:56.565 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22900 IOs, 9950867456 bytes 2026-02-22T21:49:56.598 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23000 IOs, 9963884544 bytes 2026-02-22T21:49:56.628 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23100 IOs, 9973994496 bytes 2026-02-22T21:49:56.679 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23200 IOs, 10023629824 bytes 2026-02-22T21:49:56.735 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23300 IOs, 10067934208 bytes 2026-02-22T21:49:56.791 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23400 IOs, 10117192704 bytes 2026-02-22T21:49:56.822 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23500 IOs, 10121993216 bytes 2026-02-22T21:49:56.870 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23600 IOs, 10166929408 bytes 2026-02-22T21:49:56.910 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23700 IOs, 10196435968 bytes 2026-02-22T21:49:56.942 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23800 IOs, 10209530880 bytes 2026-02-22T21:49:57.000 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23900 IOs, 10278946816 bytes 2026-02-22T21:49:57.068 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24000 IOs, 10379987968 bytes 2026-02-22T21:49:57.134 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24100 IOs, 10427468800 bytes 2026-02-22T21:49:57.213 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24200 IOs, 10530301952 bytes 2026-02-22T21:49:57.250 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24300 IOs, 10561564672 bytes 2026-02-22T21:49:57.297 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24400 IOs, 10594972672 bytes 2026-02-22T21:49:57.354 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24500 IOs, 10665769984 bytes 2026-02-22T21:49:57.407 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24600 IOs, 10717776896 bytes 2026-02-22T21:49:57.449 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24700 IOs, 10745698304 bytes 2026-02-22T21:49:57.476 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24800 IOs, 10760483840 bytes 2026-02-22T21:49:57.550 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24900 IOs, 10865709056 bytes 2026-02-22T21:49:57.618 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25000 IOs, 10932722688 bytes 2026-02-22T21:49:57.661 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25100 IOs, 10976101376 bytes 2026-02-22T21:49:57.706 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25200 IOs, 11004864512 bytes 2026-02-22T21:49:57.741 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25300 IOs, 11013675008 bytes 2026-02-22T21:49:57.811 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25400 IOs, 11096542208 bytes 2026-02-22T21:49:57.847 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25500 IOs, 11109885952 bytes 2026-02-22T21:49:57.885 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25600 IOs, 11134883840 bytes 2026-02-22T21:49:57.923 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25700 IOs, 11162918912 bytes 2026-02-22T21:49:58.000 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25800 IOs, 11258702848 bytes 2026-02-22T21:49:58.055 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25900 IOs, 11341668352 bytes 2026-02-22T21:49:58.102 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:26000 IOs, 11365065728 bytes 2026-02-22T21:49:58.150 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:26100 IOs, 11392047104 bytes 2026-02-22T21:49:58.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:26200 IOs, 11435522048 bytes 2026-02-22T21:49:58.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:26300 IOs, 11496921088 bytes 2026-02-22T21:49:58.298 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:26400 IOs, 11545750528 bytes 2026-02-22T21:49:58.348 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:26500 IOs, 11571417088 bytes 2026-02-22T21:49:58.385 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:26600 IOs, 11595558912 bytes 2026-02-22T21:49:58.422 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:26700 IOs, 11621952512 bytes 2026-02-22T21:49:58.453 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:26800 IOs, 11629608960 bytes 2026-02-22T21:49:58.504 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:26900 IOs, 11674551296 bytes 2026-02-22T21:49:58.552 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:27000 IOs, 11718366208 bytes 2026-02-22T21:49:58.618 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:27100 IOs, 11789254656 bytes 2026-02-22T21:49:58.670 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:27200 IOs, 11844259840 bytes 2026-02-22T21:49:58.694 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:27300 IOs, 11844864000 bytes 2026-02-22T21:49:58.742 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:27400 IOs, 11886317568 bytes 2026-02-22T21:49:58.800 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:27500 IOs, 11950061568 bytes 2026-02-22T21:49:58.832 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:27600 IOs, 11950669824 bytes 2026-02-22T21:49:58.858 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:27700 IOs, 11951306752 bytes 2026-02-22T21:49:58.902 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:27800 IOs, 11984852992 bytes 2026-02-22T21:49:58.940 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:27900 IOs, 12004405248 bytes 2026-02-22T21:49:58.983 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:28000 IOs, 12045199360 bytes 2026-02-22T21:49:59.028 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:28100 IOs, 12078491648 bytes 2026-02-22T21:49:59.083 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:28200 IOs, 12137628672 bytes 2026-02-22T21:49:59.117 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:28300 IOs, 12163102720 bytes 2026-02-22T21:49:59.157 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ error_code=124 2026-02-22T21:49:59.157 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set -e 2026-02-22T21:49:59.157 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' 124 -eq 124 ']' 2026-02-22T21:49:59.157 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:49:59.157 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T21:49:59.157 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_13 2026-02-22T21:49:59.158 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image_and_enable_mirror cluster2 mirror image_13 snapshot 128M 2026-02-22T21:49:59.158 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:49:59.158 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:49:59.158 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:49:59.158 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:49:59.158 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_13 2026-02-22T21:49:59.158 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:49:59.158 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T21:49:59.158 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n snapshot ']' 2026-02-22T21:49:59.158 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:49:59.158 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image cluster2 mirror image_13 128M 2026-02-22T21:49:59.158 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:49:59.159 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:49:59.159 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:49:59.159 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:49:59.159 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_13 2026-02-22T21:49:59.159 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:49:59.159 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local size=128 2026-02-22T21:49:59.159 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n 128M ']' 2026-02-22T21:49:59.159 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ size=128M 2026-02-22T21:49:59.159 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:49:59.159 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_13 2026-02-22T21:49:59.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T21:49:59.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:49:59.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' image = image ']' 2026-02-22T21:49:59.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:49:59.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ enable_mirror cluster2 mirror image_13 snapshot 2026-02-22T21:49:59.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:49:59.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:49:59.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_13 2026-02-22T21:49:59.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T21:49:59.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_13 snapshot 2026-02-22T21:50:00.123 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Mirroring enabled 2026-02-22T21:50:00.128 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 info mirror/image_13 2026-02-22T21:50:00.152 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_13': 2026-02-22T21:50:00.153 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 128 MiB in 32 objects 2026-02-22T21:50:00.153 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T21:50:00.153 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 1 2026-02-22T21:50:00.153 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 12fc6c1a8225 2026-02-22T21:50:00.153 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.12fc6c1a8225 2026-02-22T21:50:00.153 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T21:50:00.153 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock 2026-02-22T21:50:00.153 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T21:50:00.153 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T21:50:00.153 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:49:59 2026 2026-02-22T21:50:00.154 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:49:59 2026 2026-02-22T21:50:00.154 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:49:59 2026 2026-02-22T21:50:00.154 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T21:50:00.154 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T21:50:00.154 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: 5d0b4c97-6ffd-4ecc-87a0-5cb43cf91b88 2026-02-22T21:50:00.154 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: true 2026-02-22T21:50:00.155 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n '' ']' 2026-02-22T21:50:00.155 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ stress_write_image cluster2 mirror image_13 2026-02-22T21:50:00.155 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:50:00.155 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:50:00.155 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_13 2026-02-22T21:50:00.155 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-22T21:50:00.156 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local duration=9 2026-02-22T21:50:00.156 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set +e 2026-02-22T21:50:00.156 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ timeout 9s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_13 --debug-rbd=20 --debug-journaler=20 2026-02-22T21:50:00.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:100 IOs, 88740864 bytes 2026-02-22T21:50:00.325 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:200 IOs, 178823168 bytes 2026-02-22T21:50:00.356 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:300 IOs, 189782016 bytes 2026-02-22T21:50:00.398 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:400 IOs, 244241408 bytes 2026-02-22T21:50:00.451 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:500 IOs, 274214912 bytes 2026-02-22T21:50:00.501 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:600 IOs, 321336320 bytes 2026-02-22T21:50:00.530 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:700 IOs, 324995072 bytes 2026-02-22T21:50:00.562 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:800 IOs, 345210880 bytes 2026-02-22T21:50:00.605 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:900 IOs, 400139264 bytes 2026-02-22T21:50:00.660 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1000 IOs, 441125888 bytes 2026-02-22T21:50:00.690 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1100 IOs, 445237248 bytes 2026-02-22T21:50:00.739 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1200 IOs, 493702144 bytes 2026-02-22T21:50:00.764 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1300 IOs, 494291968 bytes 2026-02-22T21:50:00.820 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1400 IOs, 552892416 bytes 2026-02-22T21:50:00.871 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1500 IOs, 594007040 bytes 2026-02-22T21:50:00.920 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1600 IOs, 652581888 bytes 2026-02-22T21:50:00.952 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1700 IOs, 658752512 bytes 2026-02-22T21:50:00.982 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1800 IOs, 676118528 bytes 2026-02-22T21:50:01.011 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1900 IOs, 676745216 bytes 2026-02-22T21:50:01.062 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2000 IOs, 732600320 bytes 2026-02-22T21:50:01.118 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2100 IOs, 804636672 bytes 2026-02-22T21:50:01.161 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2200 IOs, 832281600 bytes 2026-02-22T21:50:01.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2300 IOs, 839947264 bytes 2026-02-22T21:50:01.238 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2400 IOs, 887859200 bytes 2026-02-22T21:50:01.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2500 IOs, 909896704 bytes 2026-02-22T21:50:01.353 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2600 IOs, 1019895808 bytes 2026-02-22T21:50:01.411 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2700 IOs, 1073783808 bytes 2026-02-22T21:50:01.445 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2800 IOs, 1091283968 bytes 2026-02-22T21:50:01.485 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2900 IOs, 1120198656 bytes 2026-02-22T21:50:01.523 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3000 IOs, 1148637184 bytes 2026-02-22T21:50:01.563 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3100 IOs, 1181501440 bytes 2026-02-22T21:50:01.630 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3200 IOs, 1260598272 bytes 2026-02-22T21:50:01.657 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3300 IOs, 1281972224 bytes 2026-02-22T21:50:01.702 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3400 IOs, 1312939008 bytes 2026-02-22T21:50:01.756 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3500 IOs, 1370439680 bytes 2026-02-22T21:50:01.796 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3600 IOs, 1401437184 bytes 2026-02-22T21:50:01.840 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3700 IOs, 1450586112 bytes 2026-02-22T21:50:01.884 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3800 IOs, 1479567360 bytes 2026-02-22T21:50:01.931 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3900 IOs, 1520790528 bytes 2026-02-22T21:50:01.966 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4000 IOs, 1547841536 bytes 2026-02-22T21:50:02.007 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4100 IOs, 1575912448 bytes 2026-02-22T21:50:02.043 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4200 IOs, 1592596480 bytes 2026-02-22T21:50:02.097 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4300 IOs, 1652150272 bytes 2026-02-22T21:50:02.173 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4400 IOs, 1753212928 bytes 2026-02-22T21:50:02.213 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4500 IOs, 1769809920 bytes 2026-02-22T21:50:02.261 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4600 IOs, 1813517312 bytes 2026-02-22T21:50:02.291 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4700 IOs, 1831732224 bytes 2026-02-22T21:50:02.362 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4800 IOs, 1927505920 bytes 2026-02-22T21:50:02.419 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4900 IOs, 1979245568 bytes 2026-02-22T21:50:02.474 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5000 IOs, 2032773120 bytes 2026-02-22T21:50:02.520 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5100 IOs, 2079947776 bytes 2026-02-22T21:50:02.560 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5200 IOs, 2110514176 bytes 2026-02-22T21:50:02.593 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5300 IOs, 2126111744 bytes 2026-02-22T21:50:02.626 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5400 IOs, 2137611264 bytes 2026-02-22T21:50:02.688 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5500 IOs, 2215811072 bytes 2026-02-22T21:50:02.726 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5600 IOs, 2236720128 bytes 2026-02-22T21:50:02.770 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5700 IOs, 2268926976 bytes 2026-02-22T21:50:02.816 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5800 IOs, 2295107584 bytes 2026-02-22T21:50:02.876 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5900 IOs, 2357026816 bytes 2026-02-22T21:50:02.924 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6000 IOs, 2398867456 bytes 2026-02-22T21:50:02.961 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6100 IOs, 2422400000 bytes 2026-02-22T21:50:03.000 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6200 IOs, 2451664896 bytes 2026-02-22T21:50:03.036 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6300 IOs, 2465550336 bytes 2026-02-22T21:50:03.075 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6400 IOs, 2490407936 bytes 2026-02-22T21:50:03.106 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6500 IOs, 2499969024 bytes 2026-02-22T21:50:03.152 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6600 IOs, 2540046336 bytes 2026-02-22T21:50:03.220 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6700 IOs, 2642277376 bytes 2026-02-22T21:50:03.298 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6800 IOs, 2748257280 bytes 2026-02-22T21:50:03.341 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6900 IOs, 2778851328 bytes 2026-02-22T21:50:03.388 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7000 IOs, 2816379904 bytes 2026-02-22T21:50:03.430 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7100 IOs, 2854228992 bytes 2026-02-22T21:50:03.485 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7200 IOs, 2914831360 bytes 2026-02-22T21:50:03.529 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7300 IOs, 2947857408 bytes 2026-02-22T21:50:03.568 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7400 IOs, 2981611520 bytes 2026-02-22T21:50:03.605 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7500 IOs, 2992904192 bytes 2026-02-22T21:50:03.651 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7600 IOs, 3037665280 bytes 2026-02-22T21:50:03.683 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7700 IOs, 3044431872 bytes 2026-02-22T21:50:03.716 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7800 IOs, 3054828544 bytes 2026-02-22T21:50:03.754 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7900 IOs, 3077113856 bytes 2026-02-22T21:50:03.802 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8000 IOs, 3122597888 bytes 2026-02-22T21:50:03.876 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8100 IOs, 3188098048 bytes 2026-02-22T21:50:03.938 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8200 IOs, 3256626176 bytes 2026-02-22T21:50:03.975 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8300 IOs, 3272548352 bytes 2026-02-22T21:50:04.021 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8400 IOs, 3314103296 bytes 2026-02-22T21:50:04.082 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8500 IOs, 3387092992 bytes 2026-02-22T21:50:04.137 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8600 IOs, 3439422464 bytes 2026-02-22T21:50:04.173 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8700 IOs, 3458528256 bytes 2026-02-22T21:50:04.220 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8800 IOs, 3498336256 bytes 2026-02-22T21:50:04.265 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8900 IOs, 3528382464 bytes 2026-02-22T21:50:04.303 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9000 IOs, 3552454656 bytes 2026-02-22T21:50:04.350 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9100 IOs, 3601792000 bytes 2026-02-22T21:50:04.409 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9200 IOs, 3664637952 bytes 2026-02-22T21:50:04.458 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9300 IOs, 3718043648 bytes 2026-02-22T21:50:04.512 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9400 IOs, 3785798656 bytes 2026-02-22T21:50:04.580 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9500 IOs, 3858286592 bytes 2026-02-22T21:50:04.619 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9600 IOs, 3887234048 bytes 2026-02-22T21:50:04.666 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9700 IOs, 3924232192 bytes 2026-02-22T21:50:04.720 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9800 IOs, 3980216320 bytes 2026-02-22T21:50:04.762 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9900 IOs, 4015478784 bytes 2026-02-22T21:50:04.818 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10000 IOs, 4057934848 bytes 2026-02-22T21:50:04.851 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10100 IOs, 4084270080 bytes 2026-02-22T21:50:04.913 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10200 IOs, 4166056960 bytes 2026-02-22T21:50:04.960 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10300 IOs, 4191310848 bytes 2026-02-22T21:50:05.011 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10400 IOs, 4247476224 bytes 2026-02-22T21:50:05.058 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10500 IOs, 4306150400 bytes 2026-02-22T21:50:05.122 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10600 IOs, 4374756352 bytes 2026-02-22T21:50:05.160 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10700 IOs, 4387272704 bytes 2026-02-22T21:50:05.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10800 IOs, 4431683584 bytes 2026-02-22T21:50:05.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10900 IOs, 4519211008 bytes 2026-02-22T21:50:05.342 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11000 IOs, 4613544960 bytes 2026-02-22T21:50:05.378 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11100 IOs, 4614159360 bytes 2026-02-22T21:50:05.412 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11200 IOs, 4631252992 bytes 2026-02-22T21:50:05.468 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11300 IOs, 4685262848 bytes 2026-02-22T21:50:05.504 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11400 IOs, 4712777728 bytes 2026-02-22T21:50:05.555 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11500 IOs, 4743503872 bytes 2026-02-22T21:50:05.788 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11600 IOs, 4789433344 bytes 2026-02-22T21:50:05.824 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11700 IOs, 4805664768 bytes 2026-02-22T21:50:05.850 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11800 IOs, 4806265856 bytes 2026-02-22T21:50:05.905 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11900 IOs, 4875905024 bytes 2026-02-22T21:50:05.960 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12000 IOs, 4937248768 bytes 2026-02-22T21:50:06.010 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12100 IOs, 4980424704 bytes 2026-02-22T21:50:06.067 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12200 IOs, 5020962816 bytes 2026-02-22T21:50:06.116 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12300 IOs, 5068901376 bytes 2026-02-22T21:50:06.157 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12400 IOs, 5093343232 bytes 2026-02-22T21:50:06.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12500 IOs, 5124114432 bytes 2026-02-22T21:50:06.231 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12600 IOs, 5129782272 bytes 2026-02-22T21:50:06.270 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12700 IOs, 5155429376 bytes 2026-02-22T21:50:06.306 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12800 IOs, 5176387584 bytes 2026-02-22T21:50:06.358 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12900 IOs, 5222452224 bytes 2026-02-22T21:50:06.403 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13000 IOs, 5262133248 bytes 2026-02-22T21:50:06.467 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13100 IOs, 5340843008 bytes 2026-02-22T21:50:06.501 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13200 IOs, 5341465600 bytes 2026-02-22T21:50:06.552 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13300 IOs, 5385159680 bytes 2026-02-22T21:50:06.584 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13400 IOs, 5391220736 bytes 2026-02-22T21:50:06.637 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13500 IOs, 5448809472 bytes 2026-02-22T21:50:06.679 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13600 IOs, 5474234368 bytes 2026-02-22T21:50:06.719 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13700 IOs, 5495721984 bytes 2026-02-22T21:50:06.777 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13800 IOs, 5558498304 bytes 2026-02-22T21:50:06.828 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13900 IOs, 5594792960 bytes 2026-02-22T21:50:06.875 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14000 IOs, 5636499456 bytes 2026-02-22T21:50:06.905 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14100 IOs, 5637124096 bytes 2026-02-22T21:50:06.945 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14200 IOs, 5664921600 bytes 2026-02-22T21:50:06.989 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14300 IOs, 5697653760 bytes 2026-02-22T21:50:07.063 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14400 IOs, 5805979648 bytes 2026-02-22T21:50:07.124 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14500 IOs, 5893511168 bytes 2026-02-22T21:50:07.162 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14600 IOs, 5911189504 bytes 2026-02-22T21:50:07.203 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14700 IOs, 5973037056 bytes 2026-02-22T21:50:07.243 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14800 IOs, 6000859136 bytes 2026-02-22T21:50:07.285 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14900 IOs, 6037311488 bytes 2026-02-22T21:50:07.332 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15000 IOs, 6089416704 bytes 2026-02-22T21:50:07.373 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15100 IOs, 6101372928 bytes 2026-02-22T21:50:07.448 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15200 IOs, 6203773952 bytes 2026-02-22T21:50:07.495 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15300 IOs, 6250263552 bytes 2026-02-22T21:50:07.535 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15400 IOs, 6277688320 bytes 2026-02-22T21:50:07.599 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15500 IOs, 6354428928 bytes 2026-02-22T21:50:07.666 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15600 IOs, 6445510656 bytes 2026-02-22T21:50:07.696 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15700 IOs, 6452777984 bytes 2026-02-22T21:50:07.744 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15800 IOs, 6495779840 bytes 2026-02-22T21:50:07.785 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15900 IOs, 6526594048 bytes 2026-02-22T21:50:07.843 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16000 IOs, 6573227008 bytes 2026-02-22T21:50:07.896 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16100 IOs, 6616592384 bytes 2026-02-22T21:50:07.930 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16200 IOs, 6634211328 bytes 2026-02-22T21:50:07.970 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16300 IOs, 6656882688 bytes 2026-02-22T21:50:08.010 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16400 IOs, 6690100224 bytes 2026-02-22T21:50:08.066 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16500 IOs, 6749889536 bytes 2026-02-22T21:50:08.107 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16600 IOs, 6781050880 bytes 2026-02-22T21:50:08.159 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16700 IOs, 6827614208 bytes 2026-02-22T21:50:08.184 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16800 IOs, 6828230656 bytes 2026-02-22T21:50:08.246 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16900 IOs, 6916134912 bytes 2026-02-22T21:50:08.282 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17000 IOs, 6933283840 bytes 2026-02-22T21:50:08.309 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17100 IOs, 6933915648 bytes 2026-02-22T21:50:08.356 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17200 IOs, 6977137664 bytes 2026-02-22T21:50:08.399 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17300 IOs, 7020083200 bytes 2026-02-22T21:50:08.471 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17400 IOs, 7114461184 bytes 2026-02-22T21:50:08.503 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17500 IOs, 7116253184 bytes 2026-02-22T21:50:08.528 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17600 IOs, 7116855296 bytes 2026-02-22T21:50:08.570 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17700 IOs, 7150031872 bytes 2026-02-22T21:50:08.602 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17800 IOs, 7159706624 bytes 2026-02-22T21:50:08.653 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17900 IOs, 7216030720 bytes 2026-02-22T21:50:08.693 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18000 IOs, 7245831168 bytes 2026-02-22T21:50:08.728 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18100 IOs, 7265217536 bytes 2026-02-22T21:50:08.782 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18200 IOs, 7324996608 bytes 2026-02-22T21:50:08.849 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18300 IOs, 7393655808 bytes 2026-02-22T21:50:08.892 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18400 IOs, 7425455104 bytes 2026-02-22T21:50:08.930 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18500 IOs, 7447250944 bytes 2026-02-22T21:50:08.959 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18600 IOs, 7451557888 bytes 2026-02-22T21:50:09.009 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18700 IOs, 7508255744 bytes 2026-02-22T21:50:09.071 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18800 IOs, 7571420160 bytes 2026-02-22T21:50:09.124 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18900 IOs, 7634244608 bytes 2026-02-22T21:50:09.162 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ error_code=124 2026-02-22T21:50:09.162 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set -e 2026-02-22T21:50:09.162 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' 124 -eq 124 ']' 2026-02-22T21:50:09.162 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:50:09.162 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T21:50:09.162 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_14 2026-02-22T21:50:09.163 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image_and_enable_mirror cluster2 mirror image_14 snapshot 128M 2026-02-22T21:50:09.163 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:50:09.163 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:50:09.163 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:50:09.163 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:50:09.163 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_14 2026-02-22T21:50:09.163 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:50:09.163 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T21:50:09.163 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n snapshot ']' 2026-02-22T21:50:09.163 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:50:09.163 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image cluster2 mirror image_14 128M 2026-02-22T21:50:09.163 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:50:09.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:50:09.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:50:09.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:50:09.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_14 2026-02-22T21:50:09.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:50:09.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local size=128 2026-02-22T21:50:09.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n 128M ']' 2026-02-22T21:50:09.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ size=128M 2026-02-22T21:50:09.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:50:09.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_14 2026-02-22T21:50:09.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T21:50:09.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:50:09.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' image = image ']' 2026-02-22T21:50:09.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:50:09.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ enable_mirror cluster2 mirror image_14 snapshot 2026-02-22T21:50:09.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:50:09.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:50:09.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_14 2026-02-22T21:50:09.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T21:50:09.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_14 snapshot 2026-02-22T21:50:10.142 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Mirroring enabled 2026-02-22T21:50:10.147 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 info mirror/image_14 2026-02-22T21:50:10.173 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_14': 2026-02-22T21:50:10.173 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 128 MiB in 32 objects 2026-02-22T21:50:10.173 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T21:50:10.173 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 1 2026-02-22T21:50:10.173 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 1306842dd273 2026-02-22T21:50:10.173 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.1306842dd273 2026-02-22T21:50:10.173 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T21:50:10.174 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock 2026-02-22T21:50:10.174 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T21:50:10.174 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T21:50:10.174 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:50:09 2026 2026-02-22T21:50:10.174 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:50:09 2026 2026-02-22T21:50:10.174 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:50:09 2026 2026-02-22T21:50:10.174 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T21:50:10.174 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T21:50:10.174 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: adaf312a-e129-41dc-a0bd-6d745384fd91 2026-02-22T21:50:10.174 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: true 2026-02-22T21:50:10.175 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n '' ']' 2026-02-22T21:50:10.175 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ stress_write_image cluster2 mirror image_14 2026-02-22T21:50:10.175 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:50:10.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:50:10.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_14 2026-02-22T21:50:10.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-22T21:50:10.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local duration=9 2026-02-22T21:50:10.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set +e 2026-02-22T21:50:10.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ timeout 9s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_14 --debug-rbd=20 --debug-journaler=20 2026-02-22T21:50:10.248 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:100 IOs, 24654848 bytes 2026-02-22T21:50:10.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:200 IOs, 46681088 bytes 2026-02-22T21:50:10.339 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:300 IOs, 98542592 bytes 2026-02-22T21:50:10.399 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:400 IOs, 168122368 bytes 2026-02-22T21:50:10.455 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:500 IOs, 226664448 bytes 2026-02-22T21:50:10.516 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:600 IOs, 298501120 bytes 2026-02-22T21:50:10.554 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:700 IOs, 322395136 bytes 2026-02-22T21:50:10.597 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:800 IOs, 358218752 bytes 2026-02-22T21:50:10.636 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:900 IOs, 394243072 bytes 2026-02-22T21:50:10.680 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1000 IOs, 425362432 bytes 2026-02-22T21:50:10.741 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1100 IOs, 492690432 bytes 2026-02-22T21:50:10.783 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1200 IOs, 530995200 bytes 2026-02-22T21:50:10.844 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1300 IOs, 588972032 bytes 2026-02-22T21:50:10.898 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1400 IOs, 642109440 bytes 2026-02-22T21:50:10.942 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1500 IOs, 679778304 bytes 2026-02-22T21:50:10.990 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1600 IOs, 723713024 bytes 2026-02-22T21:50:11.040 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1700 IOs, 767896576 bytes 2026-02-22T21:50:11.070 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1800 IOs, 768503808 bytes 2026-02-22T21:50:11.115 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1900 IOs, 814031872 bytes 2026-02-22T21:50:11.175 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2000 IOs, 882236416 bytes 2026-02-22T21:50:11.229 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2100 IOs, 944737280 bytes 2026-02-22T21:50:11.267 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2200 IOs, 969367552 bytes 2026-02-22T21:50:11.296 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2300 IOs, 971431936 bytes 2026-02-22T21:50:11.339 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2400 IOs, 1010492416 bytes 2026-02-22T21:50:11.377 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2500 IOs, 1030545408 bytes 2026-02-22T21:50:11.421 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2600 IOs, 1070341120 bytes 2026-02-22T21:50:11.470 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2700 IOs, 1101638656 bytes 2026-02-22T21:50:11.537 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2800 IOs, 1194774528 bytes 2026-02-22T21:50:11.585 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2900 IOs, 1237079040 bytes 2026-02-22T21:50:11.650 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3000 IOs, 1309617152 bytes 2026-02-22T21:50:11.700 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3100 IOs, 1352896512 bytes 2026-02-22T21:50:11.756 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3200 IOs, 1415094272 bytes 2026-02-22T21:50:11.824 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3300 IOs, 1532572672 bytes 2026-02-22T21:50:11.905 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3400 IOs, 1613208576 bytes 2026-02-22T21:50:11.936 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3500 IOs, 1613802496 bytes 2026-02-22T21:50:11.962 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3600 IOs, 1619372032 bytes 2026-02-22T21:50:12.020 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3700 IOs, 1676998656 bytes 2026-02-22T21:50:12.078 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3800 IOs, 1749486592 bytes 2026-02-22T21:50:12.129 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3900 IOs, 1806316544 bytes 2026-02-22T21:50:12.174 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4000 IOs, 1864788992 bytes 2026-02-22T21:50:12.241 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4100 IOs, 1923137536 bytes 2026-02-22T21:50:12.299 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4200 IOs, 1988004864 bytes 2026-02-22T21:50:12.344 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4300 IOs, 2021531648 bytes 2026-02-22T21:50:12.393 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4400 IOs, 2062732288 bytes 2026-02-22T21:50:12.453 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4500 IOs, 2140220416 bytes 2026-02-22T21:50:12.509 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4600 IOs, 2196375552 bytes 2026-02-22T21:50:12.592 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4700 IOs, 2329931776 bytes 2026-02-22T21:50:12.637 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4800 IOs, 2386907136 bytes 2026-02-22T21:50:12.683 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4900 IOs, 2420452352 bytes 2026-02-22T21:50:12.746 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5000 IOs, 2495766528 bytes 2026-02-22T21:50:12.793 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5100 IOs, 2538217472 bytes 2026-02-22T21:50:12.871 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5200 IOs, 2638346240 bytes 2026-02-22T21:50:12.919 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5300 IOs, 2685609984 bytes 2026-02-22T21:50:12.957 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5400 IOs, 2702242816 bytes 2026-02-22T21:50:12.995 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5500 IOs, 2725700608 bytes 2026-02-22T21:50:13.052 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5600 IOs, 2786185216 bytes 2026-02-22T21:50:13.108 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5700 IOs, 2849068032 bytes 2026-02-22T21:50:13.166 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5800 IOs, 2904272896 bytes 2026-02-22T21:50:13.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5900 IOs, 2949115904 bytes 2026-02-22T21:50:13.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6000 IOs, 2979472384 bytes 2026-02-22T21:50:13.300 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6100 IOs, 3017665536 bytes 2026-02-22T21:50:13.358 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6200 IOs, 3094441984 bytes 2026-02-22T21:50:13.423 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6300 IOs, 3164973056 bytes 2026-02-22T21:50:13.487 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6400 IOs, 3229786112 bytes 2026-02-22T21:50:13.528 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6500 IOs, 3254268928 bytes 2026-02-22T21:50:13.577 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6600 IOs, 3294563328 bytes 2026-02-22T21:50:13.613 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6700 IOs, 3302935552 bytes 2026-02-22T21:50:13.660 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6800 IOs, 3334591488 bytes 2026-02-22T21:50:13.696 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6900 IOs, 3352601600 bytes 2026-02-22T21:50:13.737 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7000 IOs, 3381628928 bytes 2026-02-22T21:50:13.778 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7100 IOs, 3402257408 bytes 2026-02-22T21:50:13.811 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7200 IOs, 3416433664 bytes 2026-02-22T21:50:13.855 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7300 IOs, 3441913856 bytes 2026-02-22T21:50:13.903 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7400 IOs, 3468572672 bytes 2026-02-22T21:50:13.958 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7500 IOs, 3529141248 bytes 2026-02-22T21:50:13.988 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7600 IOs, 3529783296 bytes 2026-02-22T21:50:14.026 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7700 IOs, 3551203328 bytes 2026-02-22T21:50:14.069 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7800 IOs, 3577438208 bytes 2026-02-22T21:50:14.098 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7900 IOs, 3599518720 bytes 2026-02-22T21:50:14.145 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8000 IOs, 3632886784 bytes 2026-02-22T21:50:14.184 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8100 IOs, 3633480704 bytes 2026-02-22T21:50:14.229 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8200 IOs, 3667407872 bytes 2026-02-22T21:50:14.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8300 IOs, 3689773056 bytes 2026-02-22T21:50:14.331 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8400 IOs, 3763206144 bytes 2026-02-22T21:50:14.396 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8500 IOs, 3828792320 bytes 2026-02-22T21:50:14.433 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8600 IOs, 3843526656 bytes 2026-02-22T21:50:14.670 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8700 IOs, 3855239168 bytes 2026-02-22T21:50:14.730 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8800 IOs, 3909054464 bytes 2026-02-22T21:50:14.789 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8900 IOs, 3952159744 bytes 2026-02-22T21:50:14.831 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9000 IOs, 3976470528 bytes 2026-02-22T21:50:14.866 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9100 IOs, 3981048832 bytes 2026-02-22T21:50:14.921 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9200 IOs, 4015033344 bytes 2026-02-22T21:50:14.975 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9300 IOs, 4066960384 bytes 2026-02-22T21:50:15.038 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9400 IOs, 4116098048 bytes 2026-02-22T21:50:15.073 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9500 IOs, 4128723968 bytes 2026-02-22T21:50:15.121 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9600 IOs, 4161337344 bytes 2026-02-22T21:50:15.163 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9700 IOs, 4185726976 bytes 2026-02-22T21:50:15.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9800 IOs, 4231538688 bytes 2026-02-22T21:50:15.263 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9900 IOs, 4264851456 bytes 2026-02-22T21:50:15.317 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10000 IOs, 4317606912 bytes 2026-02-22T21:50:15.348 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10100 IOs, 4318229504 bytes 2026-02-22T21:50:15.407 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10200 IOs, 4385664000 bytes 2026-02-22T21:50:15.451 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10300 IOs, 4412605440 bytes 2026-02-22T21:50:15.496 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10400 IOs, 4452636672 bytes 2026-02-22T21:50:15.556 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10500 IOs, 4504254464 bytes 2026-02-22T21:50:15.611 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10600 IOs, 4543391744 bytes 2026-02-22T21:50:15.654 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10700 IOs, 4568156160 bytes 2026-02-22T21:50:15.705 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10800 IOs, 4609031168 bytes 2026-02-22T21:50:15.754 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10900 IOs, 4639170560 bytes 2026-02-22T21:50:15.818 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11000 IOs, 4715114496 bytes 2026-02-22T21:50:15.872 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11100 IOs, 4741358592 bytes 2026-02-22T21:50:15.933 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11200 IOs, 4798955520 bytes 2026-02-22T21:50:15.973 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11300 IOs, 4806280192 bytes 2026-02-22T21:50:16.021 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11400 IOs, 4837151744 bytes 2026-02-22T21:50:16.067 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11500 IOs, 4861247488 bytes 2026-02-22T21:50:16.111 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11600 IOs, 4894541824 bytes 2026-02-22T21:50:16.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11700 IOs, 4958255104 bytes 2026-02-22T21:50:16.223 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11800 IOs, 4995957760 bytes 2026-02-22T21:50:16.289 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11900 IOs, 5071343616 bytes 2026-02-22T21:50:16.346 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12000 IOs, 5126068224 bytes 2026-02-22T21:50:16.421 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12100 IOs, 5215351808 bytes 2026-02-22T21:50:16.478 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12200 IOs, 5265487872 bytes 2026-02-22T21:50:16.516 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12300 IOs, 5281368064 bytes 2026-02-22T21:50:16.585 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12400 IOs, 5351084032 bytes 2026-02-22T21:50:16.652 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12500 IOs, 5425429504 bytes 2026-02-22T21:50:16.721 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12600 IOs, 5497903104 bytes 2026-02-22T21:50:16.752 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12700 IOs, 5504457728 bytes 2026-02-22T21:50:16.822 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12800 IOs, 5566306304 bytes 2026-02-22T21:50:16.864 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12900 IOs, 5584384000 bytes 2026-02-22T21:50:16.916 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13000 IOs, 5613487104 bytes 2026-02-22T21:50:16.959 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13100 IOs, 5635840000 bytes 2026-02-22T21:50:17.007 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13200 IOs, 5692721152 bytes 2026-02-22T21:50:17.069 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13300 IOs, 5757186048 bytes 2026-02-22T21:50:17.131 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13400 IOs, 5802047488 bytes 2026-02-22T21:50:17.183 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13500 IOs, 5842150400 bytes 2026-02-22T21:50:17.220 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13600 IOs, 5852219392 bytes 2026-02-22T21:50:17.257 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13700 IOs, 5865771008 bytes 2026-02-22T21:50:17.292 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13800 IOs, 5866365952 bytes 2026-02-22T21:50:17.340 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13900 IOs, 5902243840 bytes 2026-02-22T21:50:17.382 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14000 IOs, 5941642240 bytes 2026-02-22T21:50:17.439 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14100 IOs, 5972616192 bytes 2026-02-22T21:50:17.492 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14200 IOs, 5995427840 bytes 2026-02-22T21:50:17.527 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14300 IOs, 5996057600 bytes 2026-02-22T21:50:17.577 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14400 IOs, 6019671040 bytes 2026-02-22T21:50:17.640 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14500 IOs, 6068747264 bytes 2026-02-22T21:50:17.689 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14600 IOs, 6093818880 bytes 2026-02-22T21:50:17.779 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14700 IOs, 6198866944 bytes 2026-02-22T21:50:17.840 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14800 IOs, 6251259904 bytes 2026-02-22T21:50:17.913 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14900 IOs, 6315406336 bytes 2026-02-22T21:50:17.961 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15000 IOs, 6348194816 bytes 2026-02-22T21:50:18.014 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15100 IOs, 6365777920 bytes 2026-02-22T21:50:18.079 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15200 IOs, 6423077888 bytes 2026-02-22T21:50:18.137 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15300 IOs, 6462808064 bytes 2026-02-22T21:50:18.196 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15400 IOs, 6520840192 bytes 2026-02-22T21:50:18.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15500 IOs, 6538591232 bytes 2026-02-22T21:50:18.303 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15600 IOs, 6566976512 bytes 2026-02-22T21:50:18.368 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15700 IOs, 6623126528 bytes 2026-02-22T21:50:18.418 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15800 IOs, 6649979904 bytes 2026-02-22T21:50:18.512 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15900 IOs, 6769304576 bytes 2026-02-22T21:50:18.579 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16000 IOs, 6806848512 bytes 2026-02-22T21:50:18.636 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16100 IOs, 6848010240 bytes 2026-02-22T21:50:18.711 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16200 IOs, 6919259136 bytes 2026-02-22T21:50:18.749 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16300 IOs, 6919898112 bytes 2026-02-22T21:50:18.804 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16400 IOs, 6954464256 bytes 2026-02-22T21:50:18.852 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16500 IOs, 6973356032 bytes 2026-02-22T21:50:18.923 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16600 IOs, 7044551680 bytes 2026-02-22T21:50:18.990 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16700 IOs, 7085049856 bytes 2026-02-22T21:50:19.038 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16800 IOs, 7106744320 bytes 2026-02-22T21:50:19.099 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16900 IOs, 7155692544 bytes 2026-02-22T21:50:19.151 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17000 IOs, 7187514368 bytes 2026-02-22T21:50:19.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ error_code=124 2026-02-22T21:50:19.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set -e 2026-02-22T21:50:19.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' 124 -eq 124 ']' 2026-02-22T21:50:19.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:50:19.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T21:50:19.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_15 2026-02-22T21:50:19.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image_and_enable_mirror cluster2 mirror image_15 snapshot 128M 2026-02-22T21:50:19.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:50:19.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:50:19.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:50:19.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:50:19.183 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_15 2026-02-22T21:50:19.183 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:50:19.183 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T21:50:19.183 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n snapshot ']' 2026-02-22T21:50:19.183 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:50:19.183 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image cluster2 mirror image_15 128M 2026-02-22T21:50:19.183 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:50:19.183 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:50:19.183 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:50:19.183 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:50:19.183 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_15 2026-02-22T21:50:19.183 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:50:19.183 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local size=128 2026-02-22T21:50:19.183 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n 128M ']' 2026-02-22T21:50:19.184 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ size=128M 2026-02-22T21:50:19.184 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:50:19.184 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_15 2026-02-22T21:50:19.213 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T21:50:19.213 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:50:19.213 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' image = image ']' 2026-02-22T21:50:19.213 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:50:19.213 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ enable_mirror cluster2 mirror image_15 snapshot 2026-02-22T21:50:19.213 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:50:19.213 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:50:19.213 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_15 2026-02-22T21:50:19.214 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T21:50:19.214 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_15 snapshot 2026-02-22T21:50:20.151 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Mirroring enabled 2026-02-22T21:50:20.156 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 info mirror/image_15 2026-02-22T21:50:20.180 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_15': 2026-02-22T21:50:20.180 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 128 MiB in 32 objects 2026-02-22T21:50:20.180 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T21:50:20.180 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 1 2026-02-22T21:50:20.180 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 1311d7fd84c 2026-02-22T21:50:20.180 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.1311d7fd84c 2026-02-22T21:50:20.180 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T21:50:20.180 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock 2026-02-22T21:50:20.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T21:50:20.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T21:50:20.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:50:19 2026 2026-02-22T21:50:20.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:50:19 2026 2026-02-22T21:50:20.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:50:19 2026 2026-02-22T21:50:20.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T21:50:20.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T21:50:20.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: c476b3b3-1c03-4616-860c-bdf602d8dfcf 2026-02-22T21:50:20.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: true 2026-02-22T21:50:20.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n '' ']' 2026-02-22T21:50:20.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ stress_write_image cluster2 mirror image_15 2026-02-22T21:50:20.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:50:20.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:50:20.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_15 2026-02-22T21:50:20.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-22T21:50:20.183 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local duration=8 2026-02-22T21:50:20.183 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set +e 2026-02-22T21:50:20.183 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ timeout 8s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_15 --debug-rbd=20 --debug-journaler=20 2026-02-22T21:50:20.264 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:100 IOs, 34001920 bytes 2026-02-22T21:50:20.313 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:200 IOs, 78266368 bytes 2026-02-22T21:50:20.363 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:300 IOs, 125017088 bytes 2026-02-22T21:50:20.426 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:400 IOs, 190795776 bytes 2026-02-22T21:50:20.493 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:500 IOs, 256770048 bytes 2026-02-22T21:50:20.530 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:600 IOs, 280042496 bytes 2026-02-22T21:50:20.556 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:700 IOs, 280667136 bytes 2026-02-22T21:50:20.595 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:800 IOs, 319666176 bytes 2026-02-22T21:50:20.667 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:900 IOs, 410022912 bytes 2026-02-22T21:50:20.708 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1000 IOs, 456119296 bytes 2026-02-22T21:50:20.754 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1100 IOs, 477989888 bytes 2026-02-22T21:50:20.779 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1200 IOs, 478611456 bytes 2026-02-22T21:50:20.814 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1300 IOs, 516323328 bytes 2026-02-22T21:50:20.869 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1400 IOs, 547180544 bytes 2026-02-22T21:50:20.909 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1500 IOs, 577046528 bytes 2026-02-22T21:50:20.949 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1600 IOs, 593029120 bytes 2026-02-22T21:50:21.000 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1700 IOs, 622057472 bytes 2026-02-22T21:50:21.037 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1800 IOs, 649613312 bytes 2026-02-22T21:50:21.076 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1900 IOs, 672285696 bytes 2026-02-22T21:50:21.105 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2000 IOs, 676093952 bytes 2026-02-22T21:50:21.143 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2100 IOs, 697679872 bytes 2026-02-22T21:50:21.162 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.8608797417723585) > weight (0.25) 2026-02-22T21:50:21.162 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.4344791173491044) > weight (0.25) 2026-02-22T21:50:21.162 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.69536284049096) > weight (0.25) 2026-02-22T21:50:21.162 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.3 with skip (0.9116548231346275) > weight (0.25) 2026-02-22T21:50:21.162 INFO:tasks.rbd_mirror_thrash:waiting for 105.77070337317382 secs before thrashing 2026-02-22T21:50:21.175 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2200 IOs, 706869248 bytes 2026-02-22T21:50:21.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2300 IOs, 777038848 bytes 2026-02-22T21:50:21.282 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2400 IOs, 810179584 bytes 2026-02-22T21:50:21.317 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2500 IOs, 832048128 bytes 2026-02-22T21:50:21.360 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2600 IOs, 858482688 bytes 2026-02-22T21:50:21.408 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2700 IOs, 911324160 bytes 2026-02-22T21:50:21.466 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2800 IOs, 978082816 bytes 2026-02-22T21:50:21.521 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2900 IOs, 1006869504 bytes 2026-02-22T21:50:21.565 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3000 IOs, 1047034880 bytes 2026-02-22T21:50:21.608 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3100 IOs, 1068155904 bytes 2026-02-22T21:50:21.647 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3200 IOs, 1097320448 bytes 2026-02-22T21:50:21.679 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3300 IOs, 1126371328 bytes 2026-02-22T21:50:21.723 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3400 IOs, 1143862272 bytes 2026-02-22T21:50:21.767 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3500 IOs, 1169369088 bytes 2026-02-22T21:50:21.837 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3600 IOs, 1264272384 bytes 2026-02-22T21:50:21.884 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3700 IOs, 1315087360 bytes 2026-02-22T21:50:21.958 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3800 IOs, 1387608064 bytes 2026-02-22T21:50:22.004 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3900 IOs, 1420302336 bytes 2026-02-22T21:50:22.057 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4000 IOs, 1487160320 bytes 2026-02-22T21:50:22.107 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4100 IOs, 1520360448 bytes 2026-02-22T21:50:22.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4200 IOs, 1580648448 bytes 2026-02-22T21:50:22.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4300 IOs, 1607990272 bytes 2026-02-22T21:50:22.269 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4400 IOs, 1664367616 bytes 2026-02-22T21:50:22.327 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4500 IOs, 1730393088 bytes 2026-02-22T21:50:22.367 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4600 IOs, 1751682048 bytes 2026-02-22T21:50:22.419 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4700 IOs, 1796951040 bytes 2026-02-22T21:50:22.457 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4800 IOs, 1816996864 bytes 2026-02-22T21:50:22.505 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4900 IOs, 1855668224 bytes 2026-02-22T21:50:22.557 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5000 IOs, 1891690496 bytes 2026-02-22T21:50:22.626 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5100 IOs, 1982101504 bytes 2026-02-22T21:50:22.663 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5200 IOs, 2000053248 bytes 2026-02-22T21:50:22.726 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5300 IOs, 2086902784 bytes 2026-02-22T21:50:22.771 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5400 IOs, 2103832576 bytes 2026-02-22T21:50:22.813 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5500 IOs, 2128875520 bytes 2026-02-22T21:50:22.846 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5600 IOs, 2146911232 bytes 2026-02-22T21:50:22.893 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5700 IOs, 2197276672 bytes 2026-02-22T21:50:22.966 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5800 IOs, 2287379456 bytes 2026-02-22T21:50:23.025 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5900 IOs, 2349926400 bytes 2026-02-22T21:50:23.072 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6000 IOs, 2385986560 bytes 2026-02-22T21:50:23.114 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6100 IOs, 2426815488 bytes 2026-02-22T21:50:23.188 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6200 IOs, 2524616704 bytes 2026-02-22T21:50:23.244 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6300 IOs, 2583856128 bytes 2026-02-22T21:50:23.289 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6400 IOs, 2619205632 bytes 2026-02-22T21:50:23.326 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6500 IOs, 2638427136 bytes 2026-02-22T21:50:23.373 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6600 IOs, 2673343488 bytes 2026-02-22T21:50:23.421 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6700 IOs, 2714540032 bytes 2026-02-22T21:50:23.475 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6800 IOs, 2758198272 bytes 2026-02-22T21:50:23.537 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6900 IOs, 2825352192 bytes 2026-02-22T21:50:23.578 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7000 IOs, 2844286976 bytes 2026-02-22T21:50:23.605 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7100 IOs, 2844898304 bytes 2026-02-22T21:50:23.649 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7200 IOs, 2882569216 bytes 2026-02-22T21:50:23.696 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7300 IOs, 2917719040 bytes 2026-02-22T21:50:23.747 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7400 IOs, 2973908992 bytes 2026-02-22T21:50:23.792 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7500 IOs, 3009223680 bytes 2026-02-22T21:50:23.828 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7600 IOs, 3029386240 bytes 2026-02-22T21:50:23.868 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7700 IOs, 3047407616 bytes 2026-02-22T21:50:23.905 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7800 IOs, 3072313344 bytes 2026-02-22T21:50:23.944 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7900 IOs, 3104287744 bytes 2026-02-22T21:50:23.981 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8000 IOs, 3118959616 bytes 2026-02-22T21:50:24.028 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8100 IOs, 3143141376 bytes 2026-02-22T21:50:24.073 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8200 IOs, 3180435456 bytes 2026-02-22T21:50:24.111 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8300 IOs, 3215152128 bytes 2026-02-22T21:50:24.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8400 IOs, 3262503936 bytes 2026-02-22T21:50:24.227 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8500 IOs, 3319932928 bytes 2026-02-22T21:50:24.282 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8600 IOs, 3342461952 bytes 2026-02-22T21:50:24.331 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8700 IOs, 3388351488 bytes 2026-02-22T21:50:24.372 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8800 IOs, 3414635520 bytes 2026-02-22T21:50:24.431 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8900 IOs, 3466945536 bytes 2026-02-22T21:50:24.481 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9000 IOs, 3509054464 bytes 2026-02-22T21:50:24.538 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9100 IOs, 3577575424 bytes 2026-02-22T21:50:24.592 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9200 IOs, 3626059776 bytes 2026-02-22T21:50:24.642 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9300 IOs, 3674763264 bytes 2026-02-22T21:50:24.710 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9400 IOs, 3761927168 bytes 2026-02-22T21:50:24.773 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9500 IOs, 3827433472 bytes 2026-02-22T21:50:24.818 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9600 IOs, 3865881600 bytes 2026-02-22T21:50:24.853 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9700 IOs, 3874045952 bytes 2026-02-22T21:50:24.896 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9800 IOs, 3904992256 bytes 2026-02-22T21:50:24.936 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9900 IOs, 3938535424 bytes 2026-02-22T21:50:24.986 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10000 IOs, 3959926784 bytes 2026-02-22T21:50:25.015 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10100 IOs, 3971329024 bytes 2026-02-22T21:50:25.049 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10200 IOs, 3982521344 bytes 2026-02-22T21:50:25.096 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10300 IOs, 4019922944 bytes 2026-02-22T21:50:25.140 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10400 IOs, 4058876928 bytes 2026-02-22T21:50:25.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10500 IOs, 4102438912 bytes 2026-02-22T21:50:25.255 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10600 IOs, 4163550208 bytes 2026-02-22T21:50:25.321 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10700 IOs, 4266525696 bytes 2026-02-22T21:50:25.370 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10800 IOs, 4304184320 bytes 2026-02-22T21:50:25.411 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10900 IOs, 4315942912 bytes 2026-02-22T21:50:25.463 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11000 IOs, 4376024064 bytes 2026-02-22T21:50:25.506 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11100 IOs, 4401938432 bytes 2026-02-22T21:50:25.560 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11200 IOs, 4485895168 bytes 2026-02-22T21:50:25.608 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11300 IOs, 4532099072 bytes 2026-02-22T21:50:25.652 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11400 IOs, 4539442176 bytes 2026-02-22T21:50:25.683 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11500 IOs, 4550216704 bytes 2026-02-22T21:50:25.721 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11600 IOs, 4567504896 bytes 2026-02-22T21:50:25.784 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11700 IOs, 4638972928 bytes 2026-02-22T21:50:25.831 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11800 IOs, 4684573696 bytes 2026-02-22T21:50:25.881 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11900 IOs, 4733799424 bytes 2026-02-22T21:50:25.954 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12000 IOs, 4812833792 bytes 2026-02-22T21:50:25.999 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12100 IOs, 4819340288 bytes 2026-02-22T21:50:26.059 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12200 IOs, 4877432832 bytes 2026-02-22T21:50:26.090 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12300 IOs, 4878060544 bytes 2026-02-22T21:50:26.133 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12400 IOs, 4908633088 bytes 2026-02-22T21:50:26.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12500 IOs, 4920784896 bytes 2026-02-22T21:50:26.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12600 IOs, 4921397248 bytes 2026-02-22T21:50:26.222 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12700 IOs, 4922020864 bytes 2026-02-22T21:50:26.255 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12800 IOs, 4933981184 bytes 2026-02-22T21:50:26.312 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12900 IOs, 4978659328 bytes 2026-02-22T21:50:26.355 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13000 IOs, 5010030592 bytes 2026-02-22T21:50:26.402 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13100 IOs, 5045774336 bytes 2026-02-22T21:50:26.459 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13200 IOs, 5113689088 bytes 2026-02-22T21:50:26.509 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13300 IOs, 5152152576 bytes 2026-02-22T21:50:26.546 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13400 IOs, 5165648896 bytes 2026-02-22T21:50:26.592 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13500 IOs, 5201652736 bytes 2026-02-22T21:50:26.640 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13600 IOs, 5236935680 bytes 2026-02-22T21:50:26.691 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13700 IOs, 5308001280 bytes 2026-02-22T21:50:26.757 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13800 IOs, 5365776384 bytes 2026-02-22T21:50:26.813 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13900 IOs, 5420847104 bytes 2026-02-22T21:50:26.866 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14000 IOs, 5467841536 bytes 2026-02-22T21:50:26.951 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14100 IOs, 5586998272 bytes 2026-02-22T21:50:27.011 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14200 IOs, 5622114304 bytes 2026-02-22T21:50:27.053 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14300 IOs, 5654858752 bytes 2026-02-22T21:50:27.110 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14400 IOs, 5727683584 bytes 2026-02-22T21:50:27.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14500 IOs, 5773320192 bytes 2026-02-22T21:50:27.196 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14600 IOs, 5785223168 bytes 2026-02-22T21:50:27.267 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14700 IOs, 5878299648 bytes 2026-02-22T21:50:27.300 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14800 IOs, 5887920128 bytes 2026-02-22T21:50:27.332 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14900 IOs, 5911102464 bytes 2026-02-22T21:50:27.404 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15000 IOs, 5993436160 bytes 2026-02-22T21:50:27.455 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15100 IOs, 6036701184 bytes 2026-02-22T21:50:27.485 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15200 IOs, 6037294080 bytes 2026-02-22T21:50:27.533 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15300 IOs, 6076045312 bytes 2026-02-22T21:50:27.572 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15400 IOs, 6097912832 bytes 2026-02-22T21:50:27.601 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15500 IOs, 6103316480 bytes 2026-02-22T21:50:27.655 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15600 IOs, 6153538560 bytes 2026-02-22T21:50:27.694 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15700 IOs, 6174058496 bytes 2026-02-22T21:50:27.732 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15800 IOs, 6196131840 bytes 2026-02-22T21:50:27.783 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15900 IOs, 6243597312 bytes 2026-02-22T21:50:27.843 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16000 IOs, 6311437312 bytes 2026-02-22T21:50:27.891 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16100 IOs, 6358971392 bytes 2026-02-22T21:50:27.937 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16200 IOs, 6398322688 bytes 2026-02-22T21:50:28.004 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16300 IOs, 6456278016 bytes 2026-02-22T21:50:28.066 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16400 IOs, 6521249792 bytes 2026-02-22T21:50:28.112 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16500 IOs, 6570646528 bytes 2026-02-22T21:50:28.153 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16600 IOs, 6588008448 bytes 2026-02-22T21:50:28.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ error_code=124 2026-02-22T21:50:28.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set -e 2026-02-22T21:50:28.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' 124 -eq 124 ']' 2026-02-22T21:50:28.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:50:28.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T21:50:28.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_16 2026-02-22T21:50:28.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image_and_enable_mirror cluster2 mirror image_16 snapshot 128M 2026-02-22T21:50:28.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:50:28.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:50:28.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:50:28.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:50:28.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_16 2026-02-22T21:50:28.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:50:28.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T21:50:28.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n snapshot ']' 2026-02-22T21:50:28.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:50:28.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image cluster2 mirror image_16 128M 2026-02-22T21:50:28.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:50:28.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:50:28.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:50:28.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:50:28.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_16 2026-02-22T21:50:28.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:50:28.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local size=128 2026-02-22T21:50:28.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n 128M ']' 2026-02-22T21:50:28.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ size=128M 2026-02-22T21:50:28.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:50:28.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_16 2026-02-22T21:50:28.220 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T21:50:28.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:50:28.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' image = image ']' 2026-02-22T21:50:28.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:50:28.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ enable_mirror cluster2 mirror image_16 snapshot 2026-02-22T21:50:28.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:50:28.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:50:28.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_16 2026-02-22T21:50:28.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T21:50:28.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_16 snapshot 2026-02-22T21:50:29.159 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Mirroring enabled 2026-02-22T21:50:29.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 info mirror/image_16 2026-02-22T21:50:29.188 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_16': 2026-02-22T21:50:29.188 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 128 MiB in 32 objects 2026-02-22T21:50:29.188 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T21:50:29.188 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 1 2026-02-22T21:50:29.188 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 131cf8dc625a 2026-02-22T21:50:29.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.131cf8dc625a 2026-02-22T21:50:29.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T21:50:29.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock 2026-02-22T21:50:29.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T21:50:29.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T21:50:29.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:50:28 2026 2026-02-22T21:50:29.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:50:28 2026 2026-02-22T21:50:29.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:50:28 2026 2026-02-22T21:50:29.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T21:50:29.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T21:50:29.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: 16696fcf-2c1b-4676-bcbc-e837ecbffb01 2026-02-22T21:50:29.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: true 2026-02-22T21:50:29.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n '' ']' 2026-02-22T21:50:29.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ stress_write_image cluster2 mirror image_16 2026-02-22T21:50:29.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:50:29.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:50:29.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_16 2026-02-22T21:50:29.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-22T21:50:29.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local duration=11 2026-02-22T21:50:29.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set +e 2026-02-22T21:50:29.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ timeout 11s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_16 --debug-rbd=20 --debug-journaler=20 2026-02-22T21:50:29.293 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:100 IOs, 80056320 bytes 2026-02-22T21:50:29.340 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:200 IOs, 135599104 bytes 2026-02-22T21:50:29.411 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:300 IOs, 235420672 bytes 2026-02-22T21:50:29.474 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:400 IOs, 310996992 bytes 2026-02-22T21:50:29.520 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:500 IOs, 332137472 bytes 2026-02-22T21:50:29.569 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:600 IOs, 363595776 bytes 2026-02-22T21:50:29.802 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:700 IOs, 392189952 bytes 2026-02-22T21:50:29.837 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:800 IOs, 428463104 bytes 2026-02-22T21:50:29.901 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:900 IOs, 493709312 bytes 2026-02-22T21:50:29.951 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1000 IOs, 544506880 bytes 2026-02-22T21:50:29.976 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1100 IOs, 545129472 bytes 2026-02-22T21:50:30.049 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1200 IOs, 642356224 bytes 2026-02-22T21:50:30.113 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1300 IOs, 725511168 bytes 2026-02-22T21:50:30.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1400 IOs, 777194496 bytes 2026-02-22T21:50:30.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1500 IOs, 849150976 bytes 2026-02-22T21:50:30.263 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1600 IOs, 891564032 bytes 2026-02-22T21:50:30.314 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1700 IOs, 950679552 bytes 2026-02-22T21:50:30.375 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1800 IOs, 1013722112 bytes 2026-02-22T21:50:30.409 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1900 IOs, 1027362816 bytes 2026-02-22T21:50:30.463 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2000 IOs, 1084320768 bytes 2026-02-22T21:50:30.511 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2100 IOs, 1137491968 bytes 2026-02-22T21:50:30.553 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2200 IOs, 1154703360 bytes 2026-02-22T21:50:30.613 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2300 IOs, 1216905216 bytes 2026-02-22T21:50:30.667 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2400 IOs, 1268424704 bytes 2026-02-22T21:50:30.696 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2500 IOs, 1279590400 bytes 2026-02-22T21:50:30.750 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2600 IOs, 1321865216 bytes 2026-02-22T21:50:30.791 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2700 IOs, 1347451904 bytes 2026-02-22T21:50:30.838 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2800 IOs, 1381945344 bytes 2026-02-22T21:50:30.895 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2900 IOs, 1446411264 bytes 2026-02-22T21:50:30.929 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3000 IOs, 1467302912 bytes 2026-02-22T21:50:30.967 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3100 IOs, 1479901184 bytes 2026-02-22T21:50:31.015 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3200 IOs, 1518090240 bytes 2026-02-22T21:50:31.058 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3300 IOs, 1538462720 bytes 2026-02-22T21:50:31.110 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3400 IOs, 1581153280 bytes 2026-02-22T21:50:31.148 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3500 IOs, 1622838272 bytes 2026-02-22T21:50:31.229 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3600 IOs, 1711558656 bytes 2026-02-22T21:50:31.277 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3700 IOs, 1773821952 bytes 2026-02-22T21:50:31.317 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3800 IOs, 1782858752 bytes 2026-02-22T21:50:31.373 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3900 IOs, 1848638464 bytes 2026-02-22T21:50:31.418 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4000 IOs, 1892500480 bytes 2026-02-22T21:50:31.463 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4100 IOs, 1920589824 bytes 2026-02-22T21:50:31.514 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4200 IOs, 2003694592 bytes 2026-02-22T21:50:31.586 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4300 IOs, 2064731136 bytes 2026-02-22T21:50:31.641 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4400 IOs, 2137838592 bytes 2026-02-22T21:50:31.680 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4500 IOs, 2159383552 bytes 2026-02-22T21:50:31.711 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4600 IOs, 2169489408 bytes 2026-02-22T21:50:31.783 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4700 IOs, 2270149632 bytes 2026-02-22T21:50:31.830 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4800 IOs, 2315658240 bytes 2026-02-22T21:50:31.868 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4900 IOs, 2338712576 bytes 2026-02-22T21:50:31.928 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5000 IOs, 2418319360 bytes 2026-02-22T21:50:31.972 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5100 IOs, 2461699072 bytes 2026-02-22T21:50:32.028 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5200 IOs, 2503309312 bytes 2026-02-22T21:50:32.071 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5300 IOs, 2525113344 bytes 2026-02-22T21:50:32.137 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5400 IOs, 2589651968 bytes 2026-02-22T21:50:32.184 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5500 IOs, 2632896512 bytes 2026-02-22T21:50:32.224 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5600 IOs, 2661485568 bytes 2026-02-22T21:50:32.260 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5700 IOs, 2688940032 bytes 2026-02-22T21:50:32.339 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5800 IOs, 2794341376 bytes 2026-02-22T21:50:32.385 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5900 IOs, 2821249024 bytes 2026-02-22T21:50:32.414 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6000 IOs, 2826974208 bytes 2026-02-22T21:50:32.459 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6100 IOs, 2866865152 bytes 2026-02-22T21:50:32.495 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6200 IOs, 2883125248 bytes 2026-02-22T21:50:32.549 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6300 IOs, 2934426624 bytes 2026-02-22T21:50:32.588 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6400 IOs, 2953709568 bytes 2026-02-22T21:50:32.630 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6500 IOs, 2979460096 bytes 2026-02-22T21:50:32.676 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6600 IOs, 3034609664 bytes 2026-02-22T21:50:32.740 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6700 IOs, 3097167872 bytes 2026-02-22T21:50:32.773 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6800 IOs, 3117510656 bytes 2026-02-22T21:50:32.808 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6900 IOs, 3137910784 bytes 2026-02-22T21:50:32.868 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7000 IOs, 3183751168 bytes 2026-02-22T21:50:32.909 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7100 IOs, 3217949696 bytes 2026-02-22T21:50:32.948 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7200 IOs, 3251795968 bytes 2026-02-22T21:50:32.994 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7300 IOs, 3277115392 bytes 2026-02-22T21:50:33.042 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7400 IOs, 3314610176 bytes 2026-02-22T21:50:33.112 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7500 IOs, 3385820160 bytes 2026-02-22T21:50:33.143 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7600 IOs, 3386436608 bytes 2026-02-22T21:50:33.183 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7700 IOs, 3419894784 bytes 2026-02-22T21:50:33.239 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7800 IOs, 3487669248 bytes 2026-02-22T21:50:33.280 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7900 IOs, 3521344512 bytes 2026-02-22T21:50:33.317 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8000 IOs, 3544329216 bytes 2026-02-22T21:50:33.355 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8100 IOs, 3565508608 bytes 2026-02-22T21:50:33.418 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8200 IOs, 3635641344 bytes 2026-02-22T21:50:33.491 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8300 IOs, 3718930432 bytes 2026-02-22T21:50:33.542 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8400 IOs, 3761799168 bytes 2026-02-22T21:50:33.574 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8500 IOs, 3773221888 bytes 2026-02-22T21:50:33.626 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8600 IOs, 3806672896 bytes 2026-02-22T21:50:33.672 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8700 IOs, 3849709568 bytes 2026-02-22T21:50:33.710 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8800 IOs, 3867742208 bytes 2026-02-22T21:50:33.745 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8900 IOs, 3897273344 bytes 2026-02-22T21:50:33.776 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9000 IOs, 3897908224 bytes 2026-02-22T21:50:33.829 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9100 IOs, 3951377408 bytes 2026-02-22T21:50:33.866 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9200 IOs, 3974606848 bytes 2026-02-22T21:50:33.907 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9300 IOs, 4023771136 bytes 2026-02-22T21:50:33.972 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9400 IOs, 4081357824 bytes 2026-02-22T21:50:34.008 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9500 IOs, 4097510400 bytes 2026-02-22T21:50:34.044 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9600 IOs, 4124043264 bytes 2026-02-22T21:50:34.113 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9700 IOs, 4179876864 bytes 2026-02-22T21:50:34.180 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9800 IOs, 4281033728 bytes 2026-02-22T21:50:34.229 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9900 IOs, 4312094720 bytes 2026-02-22T21:50:34.268 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10000 IOs, 4334193664 bytes 2026-02-22T21:50:34.294 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10100 IOs, 4334782464 bytes 2026-02-22T21:50:34.358 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10200 IOs, 4431325184 bytes 2026-02-22T21:50:34.398 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10300 IOs, 4454492160 bytes 2026-02-22T21:50:34.427 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10400 IOs, 4465554432 bytes 2026-02-22T21:50:34.468 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10500 IOs, 4492262400 bytes 2026-02-22T21:50:34.502 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10600 IOs, 4533305344 bytes 2026-02-22T21:50:34.573 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10700 IOs, 4607554560 bytes 2026-02-22T21:50:34.616 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10800 IOs, 4637050880 bytes 2026-02-22T21:50:34.667 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10900 IOs, 4671389696 bytes 2026-02-22T21:50:34.710 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11000 IOs, 4698432512 bytes 2026-02-22T21:50:34.758 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11100 IOs, 4748283904 bytes 2026-02-22T21:50:34.796 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11200 IOs, 4770354176 bytes 2026-02-22T21:50:34.837 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11300 IOs, 4794293248 bytes 2026-02-22T21:50:34.884 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11400 IOs, 4848393216 bytes 2026-02-22T21:50:34.937 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11500 IOs, 4882527232 bytes 2026-02-22T21:50:34.992 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11600 IOs, 4950513664 bytes 2026-02-22T21:50:35.025 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11700 IOs, 4966585344 bytes 2026-02-22T21:50:35.067 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11800 IOs, 5016750080 bytes 2026-02-22T21:50:35.125 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11900 IOs, 5051585536 bytes 2026-02-22T21:50:35.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12000 IOs, 5111055360 bytes 2026-02-22T21:50:35.225 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12100 IOs, 5125731328 bytes 2026-02-22T21:50:35.255 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12200 IOs, 5152847872 bytes 2026-02-22T21:50:35.322 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12300 IOs, 5215962112 bytes 2026-02-22T21:50:35.380 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12400 IOs, 5272019968 bytes 2026-02-22T21:50:35.426 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12500 IOs, 5335450624 bytes 2026-02-22T21:50:35.481 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12600 IOs, 5374193664 bytes 2026-02-22T21:50:35.529 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12700 IOs, 5421075456 bytes 2026-02-22T21:50:35.557 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12800 IOs, 5429957632 bytes 2026-02-22T21:50:35.589 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12900 IOs, 5434466304 bytes 2026-02-22T21:50:35.627 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13000 IOs, 5466790912 bytes 2026-02-22T21:50:35.679 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13100 IOs, 5512214528 bytes 2026-02-22T21:50:35.721 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13200 IOs, 5545290752 bytes 2026-02-22T21:50:35.749 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13300 IOs, 5547188224 bytes 2026-02-22T21:50:35.785 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13400 IOs, 5572853760 bytes 2026-02-22T21:50:35.835 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13500 IOs, 5608909824 bytes 2026-02-22T21:50:35.871 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13600 IOs, 5631068160 bytes 2026-02-22T21:50:35.913 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13700 IOs, 5664253952 bytes 2026-02-22T21:50:35.963 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13800 IOs, 5723588608 bytes 2026-02-22T21:50:36.016 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13900 IOs, 5799208960 bytes 2026-02-22T21:50:36.064 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14000 IOs, 5825034240 bytes 2026-02-22T21:50:36.132 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14100 IOs, 5900306432 bytes 2026-02-22T21:50:36.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14200 IOs, 5989293056 bytes 2026-02-22T21:50:36.248 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14300 IOs, 6032674816 bytes 2026-02-22T21:50:36.283 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14400 IOs, 6047593472 bytes 2026-02-22T21:50:36.323 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14500 IOs, 6068670464 bytes 2026-02-22T21:50:36.380 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14600 IOs, 6124607488 bytes 2026-02-22T21:50:36.425 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14700 IOs, 6157325312 bytes 2026-02-22T21:50:36.476 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14800 IOs, 6214671360 bytes 2026-02-22T21:50:36.517 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14900 IOs, 6239480832 bytes 2026-02-22T21:50:36.559 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15000 IOs, 6288498688 bytes 2026-02-22T21:50:36.612 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15100 IOs, 6320321536 bytes 2026-02-22T21:50:36.668 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15200 IOs, 6388102144 bytes 2026-02-22T21:50:36.703 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15300 IOs, 6411850752 bytes 2026-02-22T21:50:36.773 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15400 IOs, 6495208448 bytes 2026-02-22T21:50:36.822 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15500 IOs, 6538796032 bytes 2026-02-22T21:50:36.876 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15600 IOs, 6620066816 bytes 2026-02-22T21:50:36.926 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15700 IOs, 6653901824 bytes 2026-02-22T21:50:36.979 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15800 IOs, 6701203456 bytes 2026-02-22T21:50:37.017 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15900 IOs, 6725826560 bytes 2026-02-22T21:50:37.055 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16000 IOs, 6748097536 bytes 2026-02-22T21:50:37.099 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16100 IOs, 6787538944 bytes 2026-02-22T21:50:37.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16200 IOs, 6853713920 bytes 2026-02-22T21:50:37.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16300 IOs, 6889271296 bytes 2026-02-22T21:50:37.281 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16400 IOs, 6983768064 bytes 2026-02-22T21:50:37.332 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16500 IOs, 7026846720 bytes 2026-02-22T21:50:37.360 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16600 IOs, 7027452928 bytes 2026-02-22T21:50:37.414 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16700 IOs, 7086363648 bytes 2026-02-22T21:50:37.455 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16800 IOs, 7124669440 bytes 2026-02-22T21:50:37.485 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16900 IOs, 7127790592 bytes 2026-02-22T21:50:37.548 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17000 IOs, 7214212096 bytes 2026-02-22T21:50:37.591 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17100 IOs, 7242690560 bytes 2026-02-22T21:50:37.638 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17200 IOs, 7278575616 bytes 2026-02-22T21:50:37.675 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17300 IOs, 7302508544 bytes 2026-02-22T21:50:37.730 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17400 IOs, 7378657280 bytes 2026-02-22T21:50:37.790 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17500 IOs, 7437336576 bytes 2026-02-22T21:50:37.834 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17600 IOs, 7465844736 bytes 2026-02-22T21:50:37.884 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17700 IOs, 7542554624 bytes 2026-02-22T21:50:37.943 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17800 IOs, 7593583616 bytes 2026-02-22T21:50:37.998 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17900 IOs, 7660361728 bytes 2026-02-22T21:50:38.027 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18000 IOs, 7660971008 bytes 2026-02-22T21:50:38.078 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18100 IOs, 7710091264 bytes 2026-02-22T21:50:38.140 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18200 IOs, 7776190464 bytes 2026-02-22T21:50:38.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18300 IOs, 7846035456 bytes 2026-02-22T21:50:38.270 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18400 IOs, 7953807360 bytes 2026-02-22T21:50:38.323 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18500 IOs, 7994777600 bytes 2026-02-22T21:50:38.360 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18600 IOs, 8013158400 bytes 2026-02-22T21:50:38.397 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18700 IOs, 8031108096 bytes 2026-02-22T21:50:38.448 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18800 IOs, 8078987264 bytes 2026-02-22T21:50:38.482 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18900 IOs, 8111719424 bytes 2026-02-22T21:50:38.533 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19000 IOs, 8146121728 bytes 2026-02-22T21:50:38.586 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19100 IOs, 8199912448 bytes 2026-02-22T21:50:38.633 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19200 IOs, 8231955456 bytes 2026-02-22T21:50:38.692 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19300 IOs, 8300350464 bytes 2026-02-22T21:50:38.723 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19400 IOs, 8303498240 bytes 2026-02-22T21:50:38.788 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19500 IOs, 8377526272 bytes 2026-02-22T21:50:38.830 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19600 IOs, 8423160832 bytes 2026-02-22T21:50:38.888 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19700 IOs, 8474168320 bytes 2026-02-22T21:50:38.925 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19800 IOs, 8494307328 bytes 2026-02-22T21:50:38.972 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19900 IOs, 8534663168 bytes 2026-02-22T21:50:39.004 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20000 IOs, 8548589568 bytes 2026-02-22T21:50:39.047 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20100 IOs, 8586167296 bytes 2026-02-22T21:50:39.111 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20200 IOs, 8660515840 bytes 2026-02-22T21:50:39.155 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20300 IOs, 8679418880 bytes 2026-02-22T21:50:39.228 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20400 IOs, 8770577408 bytes 2026-02-22T21:50:39.273 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20500 IOs, 8813908992 bytes 2026-02-22T21:50:39.311 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20600 IOs, 8839798784 bytes 2026-02-22T21:50:39.366 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20700 IOs, 8904542208 bytes 2026-02-22T21:50:39.422 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20800 IOs, 8962781184 bytes 2026-02-22T21:50:39.456 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20900 IOs, 8977973248 bytes 2026-02-22T21:50:39.518 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21000 IOs, 9052323840 bytes 2026-02-22T21:50:39.566 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21100 IOs, 9086158848 bytes 2026-02-22T21:50:39.610 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21200 IOs, 9146008576 bytes 2026-02-22T21:50:39.675 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21300 IOs, 9181845504 bytes 2026-02-22T21:50:39.757 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21400 IOs, 9323729920 bytes 2026-02-22T21:50:39.799 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21500 IOs, 9341282304 bytes 2026-02-22T21:50:39.849 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21600 IOs, 9396092928 bytes 2026-02-22T21:50:39.878 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21700 IOs, 9396689920 bytes 2026-02-22T21:50:39.928 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21800 IOs, 9448503296 bytes 2026-02-22T21:50:39.968 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21900 IOs, 9489343488 bytes 2026-02-22T21:50:40.030 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22000 IOs, 9546362880 bytes 2026-02-22T21:50:40.092 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22100 IOs, 9612563456 bytes 2026-02-22T21:50:40.149 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22200 IOs, 9689039872 bytes 2026-02-22T21:50:40.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ error_code=124 2026-02-22T21:50:40.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set -e 2026-02-22T21:50:40.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' 124 -eq 124 ']' 2026-02-22T21:50:40.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:50:40.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T21:50:40.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_17 2026-02-22T21:50:40.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image_and_enable_mirror cluster2 mirror image_17 snapshot 128M 2026-02-22T21:50:40.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:50:40.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:50:40.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:50:40.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:50:40.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_17 2026-02-22T21:50:40.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:50:40.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T21:50:40.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n snapshot ']' 2026-02-22T21:50:40.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:50:40.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image cluster2 mirror image_17 128M 2026-02-22T21:50:40.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:50:40.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:50:40.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:50:40.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:50:40.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_17 2026-02-22T21:50:40.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:50:40.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local size=128 2026-02-22T21:50:40.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n 128M ']' 2026-02-22T21:50:40.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ size=128M 2026-02-22T21:50:40.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:50:40.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_17 2026-02-22T21:50:40.230 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T21:50:40.231 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:50:40.231 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' image = image ']' 2026-02-22T21:50:40.231 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:50:40.231 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ enable_mirror cluster2 mirror image_17 snapshot 2026-02-22T21:50:40.231 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:50:40.231 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:50:40.231 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_17 2026-02-22T21:50:40.231 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T21:50:40.231 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_17 snapshot 2026-02-22T21:50:41.108 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Mirroring enabled 2026-02-22T21:50:41.113 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 info mirror/image_17 2026-02-22T21:50:41.137 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_17': 2026-02-22T21:50:41.138 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 128 MiB in 32 objects 2026-02-22T21:50:41.138 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T21:50:41.138 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 1 2026-02-22T21:50:41.138 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 13276029db75 2026-02-22T21:50:41.138 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.13276029db75 2026-02-22T21:50:41.138 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T21:50:41.138 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock 2026-02-22T21:50:41.138 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T21:50:41.138 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T21:50:41.139 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:50:40 2026 2026-02-22T21:50:41.139 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:50:40 2026 2026-02-22T21:50:41.139 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:50:40 2026 2026-02-22T21:50:41.139 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T21:50:41.139 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T21:50:41.139 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: 0c4cedeb-aa9b-42fc-8183-74c2a5e4fd57 2026-02-22T21:50:41.139 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: true 2026-02-22T21:50:41.140 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n '' ']' 2026-02-22T21:50:41.140 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ stress_write_image cluster2 mirror image_17 2026-02-22T21:50:41.140 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:50:41.140 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:50:41.140 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_17 2026-02-22T21:50:41.141 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-22T21:50:41.141 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local duration=10 2026-02-22T21:50:41.141 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set +e 2026-02-22T21:50:41.141 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ timeout 10s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_17 --debug-rbd=20 --debug-journaler=20 2026-02-22T21:50:41.225 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:100 IOs, 44455936 bytes 2026-02-22T21:50:41.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:200 IOs, 86429696 bytes 2026-02-22T21:50:41.315 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:300 IOs, 121616384 bytes 2026-02-22T21:50:41.364 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:400 IOs, 191577088 bytes 2026-02-22T21:50:41.400 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:500 IOs, 192541696 bytes 2026-02-22T21:50:41.467 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:600 IOs, 264048640 bytes 2026-02-22T21:50:41.502 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:700 IOs, 294535168 bytes 2026-02-22T21:50:41.575 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:800 IOs, 415192064 bytes 2026-02-22T21:50:41.616 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:900 IOs, 433273856 bytes 2026-02-22T21:50:41.659 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1000 IOs, 479835136 bytes 2026-02-22T21:50:41.705 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1100 IOs, 513647616 bytes 2026-02-22T21:50:41.762 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1200 IOs, 584025088 bytes 2026-02-22T21:50:41.794 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1300 IOs, 597929984 bytes 2026-02-22T21:50:41.830 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1400 IOs, 637323264 bytes 2026-02-22T21:50:41.880 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1500 IOs, 666635264 bytes 2026-02-22T21:50:41.938 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1600 IOs, 729806848 bytes 2026-02-22T21:50:41.994 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1700 IOs, 800313344 bytes 2026-02-22T21:50:42.071 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1800 IOs, 906456064 bytes 2026-02-22T21:50:42.123 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1900 IOs, 940055552 bytes 2026-02-22T21:50:42.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2000 IOs, 983377920 bytes 2026-02-22T21:50:42.242 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2100 IOs, 1071130624 bytes 2026-02-22T21:50:42.286 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2200 IOs, 1105529856 bytes 2026-02-22T21:50:42.323 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2300 IOs, 1124554752 bytes 2026-02-22T21:50:42.378 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2400 IOs, 1199762432 bytes 2026-02-22T21:50:42.443 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2500 IOs, 1265084416 bytes 2026-02-22T21:50:42.478 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2600 IOs, 1289792512 bytes 2026-02-22T21:50:42.524 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2700 IOs, 1318878208 bytes 2026-02-22T21:50:42.573 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2800 IOs, 1366211584 bytes 2026-02-22T21:50:42.600 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2900 IOs, 1366820864 bytes 2026-02-22T21:50:42.648 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3000 IOs, 1410291712 bytes 2026-02-22T21:50:42.707 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3100 IOs, 1479412736 bytes 2026-02-22T21:50:42.747 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3200 IOs, 1521611776 bytes 2026-02-22T21:50:42.820 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3300 IOs, 1618028544 bytes 2026-02-22T21:50:42.859 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3400 IOs, 1626893312 bytes 2026-02-22T21:50:42.918 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3500 IOs, 1697761280 bytes 2026-02-22T21:50:42.968 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3600 IOs, 1737015296 bytes 2026-02-22T21:50:43.007 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3700 IOs, 1762788352 bytes 2026-02-22T21:50:43.051 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3800 IOs, 1796398080 bytes 2026-02-22T21:50:43.098 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3900 IOs, 1839186944 bytes 2026-02-22T21:50:43.134 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4000 IOs, 1850245120 bytes 2026-02-22T21:50:43.183 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4100 IOs, 1862295552 bytes 2026-02-22T21:50:43.218 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4200 IOs, 1881170944 bytes 2026-02-22T21:50:43.294 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4300 IOs, 1986953216 bytes 2026-02-22T21:50:43.342 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4400 IOs, 2038793216 bytes 2026-02-22T21:50:43.415 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4500 IOs, 2132587520 bytes 2026-02-22T21:50:43.482 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4600 IOs, 2220201984 bytes 2026-02-22T21:50:43.545 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4700 IOs, 2281240576 bytes 2026-02-22T21:50:43.587 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4800 IOs, 2314821632 bytes 2026-02-22T21:50:43.630 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4900 IOs, 2357137408 bytes 2026-02-22T21:50:43.685 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5000 IOs, 2419864576 bytes 2026-02-22T21:50:43.743 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5100 IOs, 2490397696 bytes 2026-02-22T21:50:43.774 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5200 IOs, 2505354240 bytes 2026-02-22T21:50:43.823 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5300 IOs, 2550145024 bytes 2026-02-22T21:50:43.860 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5400 IOs, 2568760320 bytes 2026-02-22T21:50:43.888 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5500 IOs, 2574849024 bytes 2026-02-22T21:50:43.920 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5600 IOs, 2602474496 bytes 2026-02-22T21:50:43.961 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5700 IOs, 2613849088 bytes 2026-02-22T21:50:44.010 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5800 IOs, 2664794112 bytes 2026-02-22T21:50:44.049 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5900 IOs, 2688297984 bytes 2026-02-22T21:50:44.107 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6000 IOs, 2767315968 bytes 2026-02-22T21:50:44.157 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6100 IOs, 2797276160 bytes 2026-02-22T21:50:44.215 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6200 IOs, 2855398400 bytes 2026-02-22T21:50:44.261 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6300 IOs, 2899282944 bytes 2026-02-22T21:50:44.302 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6400 IOs, 2917413888 bytes 2026-02-22T21:50:44.347 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6500 IOs, 2953406464 bytes 2026-02-22T21:50:44.382 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6600 IOs, 2975937536 bytes 2026-02-22T21:50:44.415 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6700 IOs, 2990266368 bytes 2026-02-22T21:50:44.465 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6800 IOs, 3031743488 bytes 2026-02-22T21:50:44.508 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6900 IOs, 3060150272 bytes 2026-02-22T21:50:44.538 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7000 IOs, 3065022464 bytes 2026-02-22T21:50:44.578 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7100 IOs, 3109360640 bytes 2026-02-22T21:50:44.662 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7200 IOs, 3213777920 bytes 2026-02-22T21:50:44.713 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7300 IOs, 3278893056 bytes 2026-02-22T21:50:44.776 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7400 IOs, 3333111808 bytes 2026-02-22T21:50:44.841 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7500 IOs, 3411476480 bytes 2026-02-22T21:50:44.878 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7600 IOs, 3430682624 bytes 2026-02-22T21:50:44.922 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7700 IOs, 3464892416 bytes 2026-02-22T21:50:44.963 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7800 IOs, 3481842688 bytes 2026-02-22T21:50:45.017 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7900 IOs, 3535855616 bytes 2026-02-22T21:50:45.069 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8000 IOs, 3595604992 bytes 2026-02-22T21:50:45.105 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8100 IOs, 3613287424 bytes 2026-02-22T21:50:45.152 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8200 IOs, 3642999808 bytes 2026-02-22T21:50:45.187 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8300 IOs, 3659258880 bytes 2026-02-22T21:50:45.237 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8400 IOs, 3716054016 bytes 2026-02-22T21:50:45.287 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8500 IOs, 3757813760 bytes 2026-02-22T21:50:45.338 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8600 IOs, 3802153984 bytes 2026-02-22T21:50:45.380 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8700 IOs, 3835276288 bytes 2026-02-22T21:50:45.414 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8800 IOs, 3848999936 bytes 2026-02-22T21:50:45.456 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8900 IOs, 3872409600 bytes 2026-02-22T21:50:45.506 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9000 IOs, 3914718208 bytes 2026-02-22T21:50:45.540 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9100 IOs, 3928413184 bytes 2026-02-22T21:50:45.583 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9200 IOs, 3952140288 bytes 2026-02-22T21:50:45.624 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9300 IOs, 3985227776 bytes 2026-02-22T21:50:45.671 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9400 IOs, 4020425728 bytes 2026-02-22T21:50:45.698 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9500 IOs, 4021068800 bytes 2026-02-22T21:50:45.738 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9600 IOs, 4045529088 bytes 2026-02-22T21:50:45.805 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9700 IOs, 4124609536 bytes 2026-02-22T21:50:45.846 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9800 IOs, 4152159232 bytes 2026-02-22T21:50:45.889 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9900 IOs, 4190876672 bytes 2026-02-22T21:50:45.939 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10000 IOs, 4239845376 bytes 2026-02-22T21:50:45.990 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10100 IOs, 4293998592 bytes 2026-02-22T21:50:46.042 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10200 IOs, 4343837696 bytes 2026-02-22T21:50:46.070 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10300 IOs, 4344439808 bytes 2026-02-22T21:50:46.147 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10400 IOs, 4441574400 bytes 2026-02-22T21:50:46.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10500 IOs, 4518715392 bytes 2026-02-22T21:50:46.263 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10600 IOs, 4586740736 bytes 2026-02-22T21:50:46.309 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10700 IOs, 4613303296 bytes 2026-02-22T21:50:46.367 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10800 IOs, 4681769984 bytes 2026-02-22T21:50:46.407 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10900 IOs, 4702789632 bytes 2026-02-22T21:50:46.469 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11000 IOs, 4795350016 bytes 2026-02-22T21:50:46.522 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11100 IOs, 4840997888 bytes 2026-02-22T21:50:46.566 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11200 IOs, 4886191104 bytes 2026-02-22T21:50:46.616 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11300 IOs, 4931008512 bytes 2026-02-22T21:50:46.666 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11400 IOs, 4989267968 bytes 2026-02-22T21:50:46.727 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11500 IOs, 5065558016 bytes 2026-02-22T21:50:46.776 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11600 IOs, 5106331648 bytes 2026-02-22T21:50:46.810 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11700 IOs, 5120656384 bytes 2026-02-22T21:50:46.855 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11800 IOs, 5169181696 bytes 2026-02-22T21:50:46.907 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11900 IOs, 5217496064 bytes 2026-02-22T21:50:46.961 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12000 IOs, 5271641088 bytes 2026-02-22T21:50:47.004 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12100 IOs, 5309731840 bytes 2026-02-22T21:50:47.056 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12200 IOs, 5357213696 bytes 2026-02-22T21:50:47.107 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12300 IOs, 5408207872 bytes 2026-02-22T21:50:47.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12400 IOs, 5488088064 bytes 2026-02-22T21:50:47.223 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12500 IOs, 5527901184 bytes 2026-02-22T21:50:47.291 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12600 IOs, 5610150912 bytes 2026-02-22T21:50:47.348 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12700 IOs, 5661285376 bytes 2026-02-22T21:50:47.398 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12800 IOs, 5710053376 bytes 2026-02-22T21:50:47.430 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12900 IOs, 5726297088 bytes 2026-02-22T21:50:47.533 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13000 IOs, 5887289344 bytes 2026-02-22T21:50:47.573 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13100 IOs, 5924284416 bytes 2026-02-22T21:50:47.609 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13200 IOs, 5931741184 bytes 2026-02-22T21:50:47.664 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13300 IOs, 5993033728 bytes 2026-02-22T21:50:47.708 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13400 IOs, 6011922432 bytes 2026-02-22T21:50:47.737 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13500 IOs, 6014837760 bytes 2026-02-22T21:50:47.813 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13600 IOs, 6120859648 bytes 2026-02-22T21:50:47.847 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13700 IOs, 6126001152 bytes 2026-02-22T21:50:47.878 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13800 IOs, 6134598656 bytes 2026-02-22T21:50:47.932 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13900 IOs, 6198925312 bytes 2026-02-22T21:50:47.980 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14000 IOs, 6242319360 bytes 2026-02-22T21:50:48.017 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14100 IOs, 6265962496 bytes 2026-02-22T21:50:48.071 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14200 IOs, 6318570496 bytes 2026-02-22T21:50:48.122 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14300 IOs, 6372781056 bytes 2026-02-22T21:50:48.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14400 IOs, 6463457280 bytes 2026-02-22T21:50:48.233 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14500 IOs, 6483499008 bytes 2026-02-22T21:50:48.265 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14600 IOs, 6495750144 bytes 2026-02-22T21:50:48.309 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14700 IOs, 6527320064 bytes 2026-02-22T21:50:48.342 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14800 IOs, 6545549312 bytes 2026-02-22T21:50:48.389 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14900 IOs, 6584280064 bytes 2026-02-22T21:50:48.436 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15000 IOs, 6631705600 bytes 2026-02-22T21:50:48.472 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15100 IOs, 6650539008 bytes 2026-02-22T21:50:48.530 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15200 IOs, 6720343040 bytes 2026-02-22T21:50:48.599 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15300 IOs, 6813848576 bytes 2026-02-22T21:50:48.644 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15400 IOs, 6859666432 bytes 2026-02-22T21:50:48.704 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15500 IOs, 6936729600 bytes 2026-02-22T21:50:48.763 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15600 IOs, 7003051008 bytes 2026-02-22T21:50:48.819 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15700 IOs, 7071988736 bytes 2026-02-22T21:50:48.866 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15800 IOs, 7112891392 bytes 2026-02-22T21:50:48.911 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15900 IOs, 7165973504 bytes 2026-02-22T21:50:48.964 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16000 IOs, 7207280640 bytes 2026-02-22T21:50:49.017 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16100 IOs, 7257578496 bytes 2026-02-22T21:50:49.053 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16200 IOs, 7272668160 bytes 2026-02-22T21:50:49.092 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16300 IOs, 7293997056 bytes 2026-02-22T21:50:49.139 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16400 IOs, 7333306368 bytes 2026-02-22T21:50:49.185 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16500 IOs, 7365525504 bytes 2026-02-22T21:50:49.234 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16600 IOs, 7405651968 bytes 2026-02-22T21:50:49.294 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16700 IOs, 7471397888 bytes 2026-02-22T21:50:49.333 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16800 IOs, 7494750208 bytes 2026-02-22T21:50:49.389 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16900 IOs, 7564241920 bytes 2026-02-22T21:50:49.447 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17000 IOs, 7634189312 bytes 2026-02-22T21:50:49.493 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17100 IOs, 7661166592 bytes 2026-02-22T21:50:49.537 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17200 IOs, 7724411904 bytes 2026-02-22T21:50:49.585 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17300 IOs, 7748057088 bytes 2026-02-22T21:50:49.631 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17400 IOs, 7779710976 bytes 2026-02-22T21:50:49.679 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17500 IOs, 7832058880 bytes 2026-02-22T21:50:49.744 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17600 IOs, 7898357760 bytes 2026-02-22T21:50:49.775 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17700 IOs, 7906210816 bytes 2026-02-22T21:50:49.810 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17800 IOs, 7921159168 bytes 2026-02-22T21:50:49.859 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17900 IOs, 7979123712 bytes 2026-02-22T21:50:49.893 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18000 IOs, 7989763072 bytes 2026-02-22T21:50:49.926 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18100 IOs, 8005440512 bytes 2026-02-22T21:50:49.986 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18200 IOs, 8076639232 bytes 2026-02-22T21:50:50.027 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18300 IOs, 8098846720 bytes 2026-02-22T21:50:50.065 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18400 IOs, 8120231936 bytes 2026-02-22T21:50:50.110 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18500 IOs, 8159528960 bytes 2026-02-22T21:50:50.138 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18600 IOs, 8160159744 bytes 2026-02-22T21:50:50.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18700 IOs, 8233027584 bytes 2026-02-22T21:50:50.251 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18800 IOs, 8266361856 bytes 2026-02-22T21:50:50.294 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18900 IOs, 8314497024 bytes 2026-02-22T21:50:50.333 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19000 IOs, 8323331072 bytes 2026-02-22T21:50:50.380 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19100 IOs, 8366819328 bytes 2026-02-22T21:50:50.419 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19200 IOs, 8405411840 bytes 2026-02-22T21:50:50.466 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19300 IOs, 8418114560 bytes 2026-02-22T21:50:50.514 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19400 IOs, 8461481984 bytes 2026-02-22T21:50:50.565 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19500 IOs, 8518258688 bytes 2026-02-22T21:50:50.610 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19600 IOs, 8553857024 bytes 2026-02-22T21:50:50.664 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19700 IOs, 8606802944 bytes 2026-02-22T21:50:50.712 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19800 IOs, 8651034624 bytes 2026-02-22T21:50:50.750 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19900 IOs, 8672097280 bytes 2026-02-22T21:50:50.780 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20000 IOs, 8681938944 bytes 2026-02-22T21:50:50.829 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20100 IOs, 8725359616 bytes 2026-02-22T21:50:50.869 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20200 IOs, 8749522944 bytes 2026-02-22T21:50:50.907 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20300 IOs, 8779998208 bytes 2026-02-22T21:50:50.954 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20400 IOs, 8825446400 bytes 2026-02-22T21:50:50.990 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20500 IOs, 8857929728 bytes 2026-02-22T21:50:51.035 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20600 IOs, 8879243264 bytes 2026-02-22T21:50:51.070 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20700 IOs, 8898731008 bytes 2026-02-22T21:50:51.108 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20800 IOs, 8929586176 bytes 2026-02-22T21:50:51.146 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ error_code=124 2026-02-22T21:50:51.146 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set -e 2026-02-22T21:50:51.146 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' 124 -eq 124 ']' 2026-02-22T21:50:51.146 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:50:51.146 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T21:50:51.146 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_18 2026-02-22T21:50:51.147 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image_and_enable_mirror cluster2 mirror image_18 snapshot 128M 2026-02-22T21:50:51.147 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:50:51.147 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:50:51.147 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:50:51.147 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:50:51.147 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_18 2026-02-22T21:50:51.147 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:50:51.147 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T21:50:51.147 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n snapshot ']' 2026-02-22T21:50:51.147 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:50:51.147 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image cluster2 mirror image_18 128M 2026-02-22T21:50:51.147 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:50:51.147 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:50:51.147 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:50:51.147 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:50:51.147 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_18 2026-02-22T21:50:51.148 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:50:51.148 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local size=128 2026-02-22T21:50:51.148 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n 128M ']' 2026-02-22T21:50:51.148 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ size=128M 2026-02-22T21:50:51.148 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:50:51.148 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_18 2026-02-22T21:50:51.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T21:50:51.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:50:51.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' image = image ']' 2026-02-22T21:50:51.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:50:51.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ enable_mirror cluster2 mirror image_18 snapshot 2026-02-22T21:50:51.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:50:51.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:50:51.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_18 2026-02-22T21:50:51.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T21:50:51.180 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_18 snapshot 2026-02-22T21:50:52.116 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Mirroring enabled 2026-02-22T21:50:52.121 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 info mirror/image_18 2026-02-22T21:50:52.145 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_18': 2026-02-22T21:50:52.145 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 128 MiB in 32 objects 2026-02-22T21:50:52.145 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T21:50:52.145 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 1 2026-02-22T21:50:52.145 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 133143b03e8b 2026-02-22T21:50:52.146 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.133143b03e8b 2026-02-22T21:50:52.146 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T21:50:52.146 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock 2026-02-22T21:50:52.146 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T21:50:52.146 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T21:50:52.146 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:50:51 2026 2026-02-22T21:50:52.146 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:50:51 2026 2026-02-22T21:50:52.146 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:50:51 2026 2026-02-22T21:50:52.146 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T21:50:52.146 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T21:50:52.147 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: 4baf6590-1204-4386-a935-9b243b94dcfc 2026-02-22T21:50:52.147 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: true 2026-02-22T21:50:52.148 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n '' ']' 2026-02-22T21:50:52.148 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ stress_write_image cluster2 mirror image_18 2026-02-22T21:50:52.148 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:50:52.148 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:50:52.148 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_18 2026-02-22T21:50:52.148 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-22T21:50:52.149 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local duration=12 2026-02-22T21:50:52.149 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set +e 2026-02-22T21:50:52.149 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ timeout 12s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_18 --debug-rbd=20 --debug-journaler=20 2026-02-22T21:50:52.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:100 IOs, 57313280 bytes 2026-02-22T21:50:52.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:200 IOs, 98712576 bytes 2026-02-22T21:50:52.331 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:300 IOs, 149198848 bytes 2026-02-22T21:50:52.373 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:400 IOs, 176974848 bytes 2026-02-22T21:50:52.414 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:500 IOs, 207479808 bytes 2026-02-22T21:50:52.443 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:600 IOs, 208106496 bytes 2026-02-22T21:50:52.471 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:700 IOs, 209230848 bytes 2026-02-22T21:50:52.514 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:800 IOs, 243419136 bytes 2026-02-22T21:50:52.540 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:900 IOs, 244046848 bytes 2026-02-22T21:50:52.583 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1000 IOs, 281389056 bytes 2026-02-22T21:50:52.634 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1100 IOs, 342265856 bytes 2026-02-22T21:50:52.670 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1200 IOs, 363633664 bytes 2026-02-22T21:50:52.712 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1300 IOs, 419634176 bytes 2026-02-22T21:50:52.768 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1400 IOs, 481264640 bytes 2026-02-22T21:50:52.827 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1500 IOs, 541555712 bytes 2026-02-22T21:50:52.865 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1600 IOs, 562832384 bytes 2026-02-22T21:50:52.901 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1700 IOs, 582544384 bytes 2026-02-22T21:50:52.960 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1800 IOs, 651278336 bytes 2026-02-22T21:50:53.004 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1900 IOs, 688609280 bytes 2026-02-22T21:50:53.057 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2000 IOs, 731927552 bytes 2026-02-22T21:50:53.095 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2100 IOs, 761325568 bytes 2026-02-22T21:50:53.140 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2200 IOs, 805372928 bytes 2026-02-22T21:50:53.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2300 IOs, 828004352 bytes 2026-02-22T21:50:53.225 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2400 IOs, 854598656 bytes 2026-02-22T21:50:53.275 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2500 IOs, 897267712 bytes 2026-02-22T21:50:53.307 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2600 IOs, 899348480 bytes 2026-02-22T21:50:53.338 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2700 IOs, 906725376 bytes 2026-02-22T21:50:53.385 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2800 IOs, 946141184 bytes 2026-02-22T21:50:53.430 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2900 IOs, 983329792 bytes 2026-02-22T21:50:53.486 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3000 IOs, 1035337728 bytes 2026-02-22T21:50:53.534 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3100 IOs, 1082174464 bytes 2026-02-22T21:50:53.579 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3200 IOs, 1120113664 bytes 2026-02-22T21:50:53.638 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3300 IOs, 1182119936 bytes 2026-02-22T21:50:53.696 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3400 IOs, 1251336192 bytes 2026-02-22T21:50:53.755 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3500 IOs, 1309487104 bytes 2026-02-22T21:50:53.806 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3600 IOs, 1359397888 bytes 2026-02-22T21:50:53.868 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3700 IOs, 1453097984 bytes 2026-02-22T21:50:53.922 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3800 IOs, 1517350912 bytes 2026-02-22T21:50:53.951 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3900 IOs, 1517979648 bytes 2026-02-22T21:50:54.017 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4000 IOs, 1614795776 bytes 2026-02-22T21:50:54.065 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4100 IOs, 1646998528 bytes 2026-02-22T21:50:54.114 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4200 IOs, 1694306304 bytes 2026-02-22T21:50:54.161 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4300 IOs, 1762127872 bytes 2026-02-22T21:50:54.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4400 IOs, 1774481408 bytes 2026-02-22T21:50:54.248 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4500 IOs, 1796622336 bytes 2026-02-22T21:50:54.299 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4600 IOs, 1828951040 bytes 2026-02-22T21:50:54.363 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4700 IOs, 1905917952 bytes 2026-02-22T21:50:54.407 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4800 IOs, 1956084736 bytes 2026-02-22T21:50:54.440 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4900 IOs, 1967538176 bytes 2026-02-22T21:50:54.493 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5000 IOs, 2016880640 bytes 2026-02-22T21:50:54.533 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5100 IOs, 2044760064 bytes 2026-02-22T21:50:54.570 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5200 IOs, 2066811904 bytes 2026-02-22T21:50:54.635 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5300 IOs, 2162706432 bytes 2026-02-22T21:50:54.682 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5400 IOs, 2200193024 bytes 2026-02-22T21:50:54.719 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5500 IOs, 2223770624 bytes 2026-02-22T21:50:54.760 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5600 IOs, 2247248896 bytes 2026-02-22T21:50:54.797 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5700 IOs, 2281725952 bytes 2026-02-22T21:50:54.851 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5800 IOs, 2317057024 bytes 2026-02-22T21:50:54.896 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5900 IOs, 2352315392 bytes 2026-02-22T21:50:54.938 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6000 IOs, 2374226944 bytes 2026-02-22T21:50:54.994 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6100 IOs, 2434317312 bytes 2026-02-22T21:50:55.029 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6200 IOs, 2470846464 bytes 2026-02-22T21:50:55.091 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6300 IOs, 2539012096 bytes 2026-02-22T21:50:55.148 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6400 IOs, 2579822592 bytes 2026-02-22T21:50:55.186 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6500 IOs, 2612493312 bytes 2026-02-22T21:50:55.243 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6600 IOs, 2652065792 bytes 2026-02-22T21:50:55.275 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6700 IOs, 2663097344 bytes 2026-02-22T21:50:55.310 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6800 IOs, 2676117504 bytes 2026-02-22T21:50:55.365 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6900 IOs, 2724329472 bytes 2026-02-22T21:50:55.415 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7000 IOs, 2786347008 bytes 2026-02-22T21:50:55.452 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7100 IOs, 2788606976 bytes 2026-02-22T21:50:55.491 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7200 IOs, 2816991232 bytes 2026-02-22T21:50:55.543 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7300 IOs, 2874345472 bytes 2026-02-22T21:50:55.584 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7400 IOs, 2921993216 bytes 2026-02-22T21:50:55.634 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7500 IOs, 2953289728 bytes 2026-02-22T21:50:55.691 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7600 IOs, 3006686208 bytes 2026-02-22T21:50:55.739 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7700 IOs, 3055234048 bytes 2026-02-22T21:50:55.787 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7800 IOs, 3098726400 bytes 2026-02-22T21:50:55.844 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7900 IOs, 3155994624 bytes 2026-02-22T21:50:55.889 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8000 IOs, 3198456832 bytes 2026-02-22T21:50:55.940 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8100 IOs, 3257812992 bytes 2026-02-22T21:50:55.980 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8200 IOs, 3292454912 bytes 2026-02-22T21:50:56.028 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8300 IOs, 3337776128 bytes 2026-02-22T21:50:56.070 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8400 IOs, 3381332992 bytes 2026-02-22T21:50:56.112 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8500 IOs, 3406550016 bytes 2026-02-22T21:50:56.148 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8600 IOs, 3428704256 bytes 2026-02-22T21:50:56.218 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8700 IOs, 3517700096 bytes 2026-02-22T21:50:56.260 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8800 IOs, 3541349376 bytes 2026-02-22T21:50:56.321 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8900 IOs, 3597282304 bytes 2026-02-22T21:50:56.357 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9000 IOs, 3617448960 bytes 2026-02-22T21:50:56.399 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9100 IOs, 3648373760 bytes 2026-02-22T21:50:56.434 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9200 IOs, 3673324544 bytes 2026-02-22T21:50:56.486 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9300 IOs, 3723940864 bytes 2026-02-22T21:50:56.533 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9400 IOs, 3768485888 bytes 2026-02-22T21:50:56.566 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9500 IOs, 3774191616 bytes 2026-02-22T21:50:56.607 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9600 IOs, 3815464960 bytes 2026-02-22T21:50:56.655 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9700 IOs, 3849408512 bytes 2026-02-22T21:50:56.703 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9800 IOs, 3895328768 bytes 2026-02-22T21:50:56.750 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9900 IOs, 3936706560 bytes 2026-02-22T21:50:56.791 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10000 IOs, 3981325312 bytes 2026-02-22T21:50:56.841 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10100 IOs, 4020441088 bytes 2026-02-22T21:50:56.875 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10200 IOs, 4031726592 bytes 2026-02-22T21:50:56.899 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10300 IOs, 4032348160 bytes 2026-02-22T21:50:56.949 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10400 IOs, 4083239936 bytes 2026-02-22T21:50:56.989 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10500 IOs, 4109059072 bytes 2026-02-22T21:50:57.013 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10600 IOs, 4109673472 bytes 2026-02-22T21:50:57.057 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10700 IOs, 4142809088 bytes 2026-02-22T21:50:57.109 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10800 IOs, 4196134912 bytes 2026-02-22T21:50:57.186 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10900 IOs, 4308486144 bytes 2026-02-22T21:50:57.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11000 IOs, 4325263360 bytes 2026-02-22T21:50:57.280 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11100 IOs, 4378457088 bytes 2026-02-22T21:50:57.337 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11200 IOs, 4444580864 bytes 2026-02-22T21:50:57.403 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11300 IOs, 4509014016 bytes 2026-02-22T21:50:57.447 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11400 IOs, 4540020736 bytes 2026-02-22T21:50:57.480 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11500 IOs, 4559776768 bytes 2026-02-22T21:50:57.543 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11600 IOs, 4628757504 bytes 2026-02-22T21:50:57.588 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11700 IOs, 4663715840 bytes 2026-02-22T21:50:57.637 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11800 IOs, 4701716480 bytes 2026-02-22T21:50:57.678 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11900 IOs, 4730059776 bytes 2026-02-22T21:50:57.738 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12000 IOs, 4803058688 bytes 2026-02-22T21:50:57.795 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12100 IOs, 4860938240 bytes 2026-02-22T21:50:57.836 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12200 IOs, 4895840256 bytes 2026-02-22T21:50:57.915 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12300 IOs, 5006530560 bytes 2026-02-22T21:50:57.967 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12400 IOs, 5059035136 bytes 2026-02-22T21:50:58.018 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12500 IOs, 5141643264 bytes 2026-02-22T21:50:58.095 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12600 IOs, 5216857088 bytes 2026-02-22T21:50:58.140 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12700 IOs, 5256916992 bytes 2026-02-22T21:50:58.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12800 IOs, 5294257152 bytes 2026-02-22T21:50:58.225 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12900 IOs, 5305728000 bytes 2026-02-22T21:50:58.270 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13000 IOs, 5335275520 bytes 2026-02-22T21:50:58.306 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13100 IOs, 5354563584 bytes 2026-02-22T21:50:58.343 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13200 IOs, 5370222592 bytes 2026-02-22T21:50:58.386 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13300 IOs, 5394649088 bytes 2026-02-22T21:50:58.424 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13400 IOs, 5426551808 bytes 2026-02-22T21:50:58.478 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13500 IOs, 5475169280 bytes 2026-02-22T21:50:58.526 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13600 IOs, 5522806784 bytes 2026-02-22T21:50:58.589 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13700 IOs, 5587311616 bytes 2026-02-22T21:50:58.660 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13800 IOs, 5672688640 bytes 2026-02-22T21:50:58.719 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13900 IOs, 5733530624 bytes 2026-02-22T21:50:58.767 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14000 IOs, 5775422464 bytes 2026-02-22T21:50:58.805 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14100 IOs, 5796512768 bytes 2026-02-22T21:50:58.856 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14200 IOs, 5857238016 bytes 2026-02-22T21:50:58.908 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14300 IOs, 5914654720 bytes 2026-02-22T21:50:58.962 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14400 IOs, 5972987904 bytes 2026-02-22T21:50:58.992 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14500 IOs, 5990481920 bytes 2026-02-22T21:50:59.046 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14600 IOs, 6028753920 bytes 2026-02-22T21:50:59.090 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14700 IOs, 6059105280 bytes 2026-02-22T21:50:59.123 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14800 IOs, 6068870144 bytes 2026-02-22T21:50:59.174 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14900 IOs, 6117976064 bytes 2026-02-22T21:50:59.233 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15000 IOs, 6178164736 bytes 2026-02-22T21:50:59.302 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15100 IOs, 6255660032 bytes 2026-02-22T21:50:59.336 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15200 IOs, 6259737600 bytes 2026-02-22T21:50:59.384 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15300 IOs, 6295520256 bytes 2026-02-22T21:50:59.431 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15400 IOs, 6331695104 bytes 2026-02-22T21:50:59.499 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15500 IOs, 6423008256 bytes 2026-02-22T21:50:59.526 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15600 IOs, 6423631872 bytes 2026-02-22T21:50:59.591 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15700 IOs, 6500868096 bytes 2026-02-22T21:50:59.632 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15800 IOs, 6527684608 bytes 2026-02-22T21:50:59.676 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15900 IOs, 6567036928 bytes 2026-02-22T21:50:59.731 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16000 IOs, 6622422016 bytes 2026-02-22T21:50:59.772 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16100 IOs, 6653431808 bytes 2026-02-22T21:50:59.816 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16200 IOs, 6709049344 bytes 2026-02-22T21:50:59.865 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16300 IOs, 6742886400 bytes 2026-02-22T21:50:59.894 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16400 IOs, 6743886848 bytes 2026-02-22T21:50:59.949 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16500 IOs, 6809604096 bytes 2026-02-22T21:51:00.013 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16600 IOs, 6886703104 bytes 2026-02-22T21:51:00.044 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16700 IOs, 6899485696 bytes 2026-02-22T21:51:00.081 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16800 IOs, 6915170304 bytes 2026-02-22T21:51:00.125 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16900 IOs, 6952108032 bytes 2026-02-22T21:51:00.174 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17000 IOs, 6992775168 bytes 2026-02-22T21:51:00.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17100 IOs, 7032387584 bytes 2026-02-22T21:51:00.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17200 IOs, 7097931776 bytes 2026-02-22T21:51:00.315 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17300 IOs, 7115848704 bytes 2026-02-22T21:51:00.393 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17400 IOs, 7206777856 bytes 2026-02-22T21:51:00.435 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17500 IOs, 7245651968 bytes 2026-02-22T21:51:00.488 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17600 IOs, 7284798464 bytes 2026-02-22T21:51:00.538 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17700 IOs, 7327998976 bytes 2026-02-22T21:51:00.580 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17800 IOs, 7389511680 bytes 2026-02-22T21:51:00.624 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17900 IOs, 7427618816 bytes 2026-02-22T21:51:00.663 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18000 IOs, 7442545664 bytes 2026-02-22T21:51:00.710 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18100 IOs, 7483268096 bytes 2026-02-22T21:51:00.772 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18200 IOs, 7553134592 bytes 2026-02-22T21:51:00.811 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18300 IOs, 7572786176 bytes 2026-02-22T21:51:00.860 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18400 IOs, 7627671552 bytes 2026-02-22T21:51:00.902 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18500 IOs, 7653105664 bytes 2026-02-22T21:51:00.960 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18600 IOs, 7720239104 bytes 2026-02-22T21:51:00.989 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18700 IOs, 7720849408 bytes 2026-02-22T21:51:01.031 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18800 IOs, 7751002112 bytes 2026-02-22T21:51:01.079 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18900 IOs, 7790925824 bytes 2026-02-22T21:51:01.129 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19000 IOs, 7839353856 bytes 2026-02-22T21:51:01.154 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19100 IOs, 7840557056 bytes 2026-02-22T21:51:01.231 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19200 IOs, 7945793536 bytes 2026-02-22T21:51:01.302 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19300 IOs, 8037120000 bytes 2026-02-22T21:51:01.382 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19400 IOs, 8146615296 bytes 2026-02-22T21:51:01.440 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19500 IOs, 8205717504 bytes 2026-02-22T21:51:01.485 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19600 IOs, 8244035584 bytes 2026-02-22T21:51:01.512 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19700 IOs, 8244656128 bytes 2026-02-22T21:51:01.551 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19800 IOs, 8280297472 bytes 2026-02-22T21:51:01.612 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19900 IOs, 8352401408 bytes 2026-02-22T21:51:01.662 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20000 IOs, 8413405184 bytes 2026-02-22T21:51:01.699 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20100 IOs, 8439313408 bytes 2026-02-22T21:51:01.743 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20200 IOs, 8484997120 bytes 2026-02-22T21:51:01.810 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20300 IOs, 8572685312 bytes 2026-02-22T21:51:01.846 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20400 IOs, 8589240320 bytes 2026-02-22T21:51:01.905 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20500 IOs, 8667676672 bytes 2026-02-22T21:51:01.958 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20600 IOs, 8692787200 bytes 2026-02-22T21:51:02.000 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20700 IOs, 8727633920 bytes 2026-02-22T21:51:02.031 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20800 IOs, 8736858112 bytes 2026-02-22T21:51:02.094 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20900 IOs, 8806192128 bytes 2026-02-22T21:51:02.132 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21000 IOs, 8833179648 bytes 2026-02-22T21:51:02.171 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21100 IOs, 8854033408 bytes 2026-02-22T21:51:02.228 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21200 IOs, 8912806912 bytes 2026-02-22T21:51:02.259 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21300 IOs, 8913410048 bytes 2026-02-22T21:51:02.316 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21400 IOs, 8957301760 bytes 2026-02-22T21:51:02.362 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21500 IOs, 8981311488 bytes 2026-02-22T21:51:02.407 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21600 IOs, 9029307392 bytes 2026-02-22T21:51:02.455 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21700 IOs, 9064591360 bytes 2026-02-22T21:51:02.496 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21800 IOs, 9084911616 bytes 2026-02-22T21:51:02.565 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21900 IOs, 9181080576 bytes 2026-02-22T21:51:02.613 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22000 IOs, 9234870272 bytes 2026-02-22T21:51:02.659 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22100 IOs, 9250912256 bytes 2026-02-22T21:51:02.701 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22200 IOs, 9291356160 bytes 2026-02-22T21:51:02.746 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22300 IOs, 9325177856 bytes 2026-02-22T21:51:02.824 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22400 IOs, 9445742592 bytes 2026-02-22T21:51:02.887 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22500 IOs, 9515149312 bytes 2026-02-22T21:51:02.939 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22600 IOs, 9565711360 bytes 2026-02-22T21:51:02.983 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22700 IOs, 9598860288 bytes 2026-02-22T21:51:03.027 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22800 IOs, 9633573888 bytes 2026-02-22T21:51:03.067 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22900 IOs, 9654326272 bytes 2026-02-22T21:51:03.105 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23000 IOs, 9676325888 bytes 2026-02-22T21:51:03.166 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23100 IOs, 9768577024 bytes 2026-02-22T21:51:03.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23200 IOs, 9822954496 bytes 2026-02-22T21:51:03.292 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23300 IOs, 9876339712 bytes 2026-02-22T21:51:03.327 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23400 IOs, 9885785088 bytes 2026-02-22T21:51:03.375 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23500 IOs, 9927208960 bytes 2026-02-22T21:51:03.424 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23600 IOs, 9970350080 bytes 2026-02-22T21:51:03.487 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23700 IOs, 10043980800 bytes 2026-02-22T21:51:03.520 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23800 IOs, 10056283136 bytes 2026-02-22T21:51:03.551 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23900 IOs, 10058315776 bytes 2026-02-22T21:51:03.630 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24000 IOs, 10163937280 bytes 2026-02-22T21:51:03.678 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24100 IOs, 10201894912 bytes 2026-02-22T21:51:03.731 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24200 IOs, 10264454144 bytes 2026-02-22T21:51:03.803 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24300 IOs, 10353641472 bytes 2026-02-22T21:51:03.842 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24400 IOs, 10377102336 bytes 2026-02-22T21:51:03.894 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24500 IOs, 10437773312 bytes 2026-02-22T21:51:03.936 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24600 IOs, 10479945728 bytes 2026-02-22T21:51:03.965 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24700 IOs, 10480564224 bytes 2026-02-22T21:51:04.018 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24800 IOs, 10537287680 bytes 2026-02-22T21:51:04.066 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24900 IOs, 10573648896 bytes 2026-02-22T21:51:04.106 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25000 IOs, 10594573312 bytes 2026-02-22T21:51:04.135 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25100 IOs, 10600526848 bytes 2026-02-22T21:51:04.153 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ error_code=124 2026-02-22T21:51:04.153 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set -e 2026-02-22T21:51:04.153 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' 124 -eq 124 ']' 2026-02-22T21:51:04.153 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:51:04.153 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T21:51:04.153 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_19 2026-02-22T21:51:04.153 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image_and_enable_mirror cluster2 mirror image_19 snapshot 128M 2026-02-22T21:51:04.153 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:51:04.153 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:51:04.153 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:51:04.153 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:51:04.153 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_19 2026-02-22T21:51:04.153 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:51:04.153 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T21:51:04.153 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n snapshot ']' 2026-02-22T21:51:04.153 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:51:04.153 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image cluster2 mirror image_19 128M 2026-02-22T21:51:04.153 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:51:04.153 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:51:04.153 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:51:04.154 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:51:04.154 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_19 2026-02-22T21:51:04.154 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:51:04.154 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local size=128 2026-02-22T21:51:04.154 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n 128M ']' 2026-02-22T21:51:04.154 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ size=128M 2026-02-22T21:51:04.154 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:51:04.154 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_19 2026-02-22T21:51:04.184 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T21:51:04.184 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:51:04.184 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' image = image ']' 2026-02-22T21:51:04.184 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:51:04.184 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ enable_mirror cluster2 mirror image_19 snapshot 2026-02-22T21:51:04.184 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:51:04.185 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:51:04.185 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_19 2026-02-22T21:51:04.185 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T21:51:04.185 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_19 snapshot 2026-02-22T21:51:05.124 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Mirroring enabled 2026-02-22T21:51:05.129 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 info mirror/image_19 2026-02-22T21:51:05.152 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_19': 2026-02-22T21:51:05.152 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 128 MiB in 32 objects 2026-02-22T21:51:05.153 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T21:51:05.153 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 1 2026-02-22T21:51:05.153 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 133c8300a8bc 2026-02-22T21:51:05.153 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.133c8300a8bc 2026-02-22T21:51:05.153 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T21:51:05.153 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock 2026-02-22T21:51:05.153 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T21:51:05.153 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T21:51:05.153 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:51:04 2026 2026-02-22T21:51:05.153 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:51:04 2026 2026-02-22T21:51:05.153 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:51:04 2026 2026-02-22T21:51:05.153 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T21:51:05.153 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T21:51:05.153 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: 7a33a362-8595-45cc-9e4c-7b4215120e99 2026-02-22T21:51:05.153 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: true 2026-02-22T21:51:05.154 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n '' ']' 2026-02-22T21:51:05.154 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ stress_write_image cluster2 mirror image_19 2026-02-22T21:51:05.154 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:51:05.154 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:51:05.154 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_19 2026-02-22T21:51:05.154 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-22T21:51:05.156 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local duration=10 2026-02-22T21:51:05.156 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set +e 2026-02-22T21:51:05.156 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ timeout 10s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_19 --debug-rbd=20 --debug-journaler=20 2026-02-22T21:51:05.431 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:100 IOs, 34964480 bytes 2026-02-22T21:51:05.484 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:200 IOs, 93209600 bytes 2026-02-22T21:51:05.540 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:300 IOs, 153990144 bytes 2026-02-22T21:51:05.573 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:400 IOs, 166121472 bytes 2026-02-22T21:51:05.607 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:500 IOs, 181889024 bytes 2026-02-22T21:51:05.647 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:600 IOs, 215720960 bytes 2026-02-22T21:51:05.713 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:700 IOs, 298970112 bytes 2026-02-22T21:51:05.754 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:800 IOs, 323915776 bytes 2026-02-22T21:51:05.793 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:900 IOs, 347886592 bytes 2026-02-22T21:51:05.846 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1000 IOs, 397986816 bytes 2026-02-22T21:51:05.889 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1100 IOs, 438332416 bytes 2026-02-22T21:51:05.926 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1200 IOs, 459286528 bytes 2026-02-22T21:51:05.993 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1300 IOs, 564975616 bytes 2026-02-22T21:51:06.028 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1400 IOs, 573685760 bytes 2026-02-22T21:51:06.065 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1500 IOs, 595143680 bytes 2026-02-22T21:51:06.137 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1600 IOs, 695248896 bytes 2026-02-22T21:51:06.166 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1700 IOs, 696089600 bytes 2026-02-22T21:51:06.223 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1800 IOs, 757549056 bytes 2026-02-22T21:51:06.274 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1900 IOs, 809279488 bytes 2026-02-22T21:51:06.325 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2000 IOs, 862893056 bytes 2026-02-22T21:51:06.367 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2100 IOs, 894741504 bytes 2026-02-22T21:51:06.424 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2200 IOs, 951782400 bytes 2026-02-22T21:51:06.454 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2300 IOs, 971071488 bytes 2026-02-22T21:51:06.537 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2400 IOs, 1072629760 bytes 2026-02-22T21:51:06.574 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2500 IOs, 1092652032 bytes 2026-02-22T21:51:06.605 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2600 IOs, 1123076096 bytes 2026-02-22T21:51:06.670 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2700 IOs, 1200724992 bytes 2026-02-22T21:51:06.708 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2800 IOs, 1216964608 bytes 2026-02-22T21:51:06.747 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2900 IOs, 1240975360 bytes 2026-02-22T21:51:06.785 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3000 IOs, 1265449984 bytes 2026-02-22T21:51:06.843 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3100 IOs, 1334234112 bytes 2026-02-22T21:51:06.896 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3200 IOs, 1388642304 bytes 2026-02-22T21:51:06.926 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3300 IOs, 1392485376 bytes 2026-02-22T21:51:06.951 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3400 IOs, 1393118208 bytes 2026-02-22T21:51:06.999 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3500 IOs, 1436242944 bytes 2026-02-22T21:51:07.046 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3600 IOs, 1484853248 bytes 2026-02-22T21:51:07.089 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3700 IOs, 1527847936 bytes 2026-02-22T21:51:07.127 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3800 IOs, 1552678912 bytes 2026-02-22T21:51:07.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3900 IOs, 1574702080 bytes 2026-02-22T21:51:07.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4000 IOs, 1635618816 bytes 2026-02-22T21:51:07.259 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4100 IOs, 1656177664 bytes 2026-02-22T21:51:07.315 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4200 IOs, 1729264640 bytes 2026-02-22T21:51:07.365 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4300 IOs, 1756431360 bytes 2026-02-22T21:51:07.406 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4400 IOs, 1801166848 bytes 2026-02-22T21:51:07.453 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4500 IOs, 1836572672 bytes 2026-02-22T21:51:07.518 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4600 IOs, 1901303808 bytes 2026-02-22T21:51:07.552 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4700 IOs, 1915120640 bytes 2026-02-22T21:51:07.607 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4800 IOs, 1962043392 bytes 2026-02-22T21:51:07.653 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4900 IOs, 1994164224 bytes 2026-02-22T21:51:07.686 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5000 IOs, 2011114496 bytes 2026-02-22T21:51:07.731 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5100 IOs, 2059167744 bytes 2026-02-22T21:51:07.786 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.0 2026-02-22T21:51:07.786 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Sent signal 15 2026-02-22T21:51:07.787 INFO:tasks.rbd_mirror_thrash:waiting for 9.426960882991 secs before reviving daemons 2026-02-22T21:51:07.787 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5200 IOs, 2112996352 bytes 2026-02-22T21:51:07.787 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial026.stderr:2026-02-22T21:51:07.785+0000 7ff98be16640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 (PID: 58143) UID: 1000 2026-02-22T21:51:07.827 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5300 IOs, 2127128576 bytes 2026-02-22T21:51:07.873 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5400 IOs, 2159771648 bytes 2026-02-22T21:51:07.924 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5500 IOs, 2224352256 bytes 2026-02-22T21:51:07.994 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5600 IOs, 2298624000 bytes 2026-02-22T21:51:08.068 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5700 IOs, 2396689408 bytes 2026-02-22T21:51:08.114 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5800 IOs, 2438411264 bytes 2026-02-22T21:51:08.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5900 IOs, 2509743104 bytes 2026-02-22T21:51:08.209 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6000 IOs, 2523327488 bytes 2026-02-22T21:51:08.243 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6100 IOs, 2533520384 bytes 2026-02-22T21:51:08.279 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6200 IOs, 2556123136 bytes 2026-02-22T21:51:08.319 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6300 IOs, 2579425280 bytes 2026-02-22T21:51:08.350 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6400 IOs, 2584718336 bytes 2026-02-22T21:51:08.390 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6500 IOs, 2608324608 bytes 2026-02-22T21:51:08.449 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6600 IOs, 2664285184 bytes 2026-02-22T21:51:08.498 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6700 IOs, 2707272704 bytes 2026-02-22T21:51:08.531 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6800 IOs, 2718955520 bytes 2026-02-22T21:51:08.564 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6900 IOs, 2726076416 bytes 2026-02-22T21:51:08.610 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7000 IOs, 2766089216 bytes 2026-02-22T21:51:08.655 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7100 IOs, 2806559744 bytes 2026-02-22T21:51:08.706 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7200 IOs, 2851173376 bytes 2026-02-22T21:51:08.751 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7300 IOs, 2885978112 bytes 2026-02-22T21:51:08.780 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7400 IOs, 2886593536 bytes 2026-02-22T21:51:08.823 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7500 IOs, 2919905280 bytes 2026-02-22T21:51:08.864 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7600 IOs, 2941768704 bytes 2026-02-22T21:51:08.922 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7700 IOs, 3009782784 bytes 2026-02-22T21:51:08.981 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7800 IOs, 3065260032 bytes 2026-02-22T21:51:09.026 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7900 IOs, 3095784448 bytes 2026-02-22T21:51:09.057 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8000 IOs, 3099871232 bytes 2026-02-22T21:51:09.094 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8100 IOs, 3121396736 bytes 2026-02-22T21:51:09.125 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8200 IOs, 3139594240 bytes 2026-02-22T21:51:09.174 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8300 IOs, 3191762944 bytes 2026-02-22T21:51:09.229 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8400 IOs, 3238146048 bytes 2026-02-22T21:51:09.261 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8500 IOs, 3244227584 bytes 2026-02-22T21:51:09.314 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8600 IOs, 3299877888 bytes 2026-02-22T21:51:09.368 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8700 IOs, 3346155520 bytes 2026-02-22T21:51:09.435 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8800 IOs, 3424564224 bytes 2026-02-22T21:51:09.488 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8900 IOs, 3476471808 bytes 2026-02-22T21:51:09.526 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9000 IOs, 3498894336 bytes 2026-02-22T21:51:09.573 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9100 IOs, 3537111040 bytes 2026-02-22T21:51:09.616 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9200 IOs, 3565687808 bytes 2026-02-22T21:51:09.663 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9300 IOs, 3602855936 bytes 2026-02-22T21:51:09.701 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9400 IOs, 3623418880 bytes 2026-02-22T21:51:09.737 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9500 IOs, 3642515456 bytes 2026-02-22T21:51:09.773 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9600 IOs, 3658434560 bytes 2026-02-22T21:51:09.800 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9700 IOs, 3659060224 bytes 2026-02-22T21:51:09.887 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9800 IOs, 3776727040 bytes 2026-02-22T21:51:09.922 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9900 IOs, 3793310720 bytes 2026-02-22T21:51:09.979 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10000 IOs, 3868962816 bytes 2026-02-22T21:51:10.027 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10100 IOs, 3898932224 bytes 2026-02-22T21:51:10.055 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10200 IOs, 3903052800 bytes 2026-02-22T21:51:10.103 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10300 IOs, 3948039168 bytes 2026-02-22T21:51:10.148 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10400 IOs, 3972487168 bytes 2026-02-22T21:51:10.220 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10500 IOs, 4074228736 bytes 2026-02-22T21:51:10.281 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10600 IOs, 4150937600 bytes 2026-02-22T21:51:10.316 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10700 IOs, 4166334464 bytes 2026-02-22T21:51:10.366 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10800 IOs, 4202344448 bytes 2026-02-22T21:51:10.417 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10900 IOs, 4251591680 bytes 2026-02-22T21:51:10.460 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11000 IOs, 4287290368 bytes 2026-02-22T21:51:10.499 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11100 IOs, 4314925056 bytes 2026-02-22T21:51:10.556 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11200 IOs, 4350168064 bytes 2026-02-22T21:51:10.596 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11300 IOs, 4375403520 bytes 2026-02-22T21:51:10.639 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11400 IOs, 4402832384 bytes 2026-02-22T21:51:10.682 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11500 IOs, 4425819136 bytes 2026-02-22T21:51:10.716 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11600 IOs, 4440432640 bytes 2026-02-22T21:51:10.779 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11700 IOs, 4513961984 bytes 2026-02-22T21:51:10.830 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11800 IOs, 4554392576 bytes 2026-02-22T21:51:10.859 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11900 IOs, 4555025408 bytes 2026-02-22T21:51:10.889 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12000 IOs, 4556365824 bytes 2026-02-22T21:51:10.928 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12100 IOs, 4593508352 bytes 2026-02-22T21:51:10.989 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12200 IOs, 4650596352 bytes 2026-02-22T21:51:11.040 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12300 IOs, 4707154944 bytes 2026-02-22T21:51:11.089 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12400 IOs, 4754935808 bytes 2026-02-22T21:51:11.130 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12500 IOs, 4786247680 bytes 2026-02-22T21:51:11.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12600 IOs, 4805352448 bytes 2026-02-22T21:51:11.226 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12700 IOs, 4896115712 bytes 2026-02-22T21:51:11.285 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12800 IOs, 4945573888 bytes 2026-02-22T21:51:11.322 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12900 IOs, 5016793088 bytes 2026-02-22T21:51:11.408 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13000 IOs, 5081428992 bytes 2026-02-22T21:51:11.463 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13100 IOs, 5129373696 bytes 2026-02-22T21:51:11.496 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13200 IOs, 5139688448 bytes 2026-02-22T21:51:11.561 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13300 IOs, 5243884544 bytes 2026-02-22T21:51:11.598 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13400 IOs, 5251675136 bytes 2026-02-22T21:51:11.642 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13500 IOs, 5297843200 bytes 2026-02-22T21:51:11.688 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13600 IOs, 5323933696 bytes 2026-02-22T21:51:11.745 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13700 IOs, 5387656192 bytes 2026-02-22T21:51:11.793 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13800 IOs, 5459010560 bytes 2026-02-22T21:51:11.862 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13900 IOs, 5526716416 bytes 2026-02-22T21:51:11.907 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14000 IOs, 5566118912 bytes 2026-02-22T21:51:11.947 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14100 IOs, 5595314176 bytes 2026-02-22T21:51:11.998 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14200 IOs, 5644414976 bytes 2026-02-22T21:51:12.038 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14300 IOs, 5676284928 bytes 2026-02-22T21:51:12.082 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14400 IOs, 5705571328 bytes 2026-02-22T21:51:12.137 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14500 IOs, 5763037184 bytes 2026-02-22T21:51:12.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14600 IOs, 5786569728 bytes 2026-02-22T21:51:12.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14700 IOs, 5788581888 bytes 2026-02-22T21:51:12.240 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14800 IOs, 5793666048 bytes 2026-02-22T21:51:12.296 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14900 IOs, 5856708608 bytes 2026-02-22T21:51:12.339 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15000 IOs, 5898853376 bytes 2026-02-22T21:51:12.384 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15100 IOs, 5951551488 bytes 2026-02-22T21:51:12.445 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15200 IOs, 6017045504 bytes 2026-02-22T21:51:12.517 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15300 IOs, 6092753920 bytes 2026-02-22T21:51:12.568 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15400 IOs, 6146989056 bytes 2026-02-22T21:51:12.631 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15500 IOs, 6221844480 bytes 2026-02-22T21:51:12.696 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15600 IOs, 6303674368 bytes 2026-02-22T21:51:12.752 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15700 IOs, 6369257472 bytes 2026-02-22T21:51:12.819 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15800 IOs, 6468577280 bytes 2026-02-22T21:51:12.898 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15900 IOs, 6550610944 bytes 2026-02-22T21:51:12.935 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16000 IOs, 6571418624 bytes 2026-02-22T21:51:12.969 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16100 IOs, 6587584512 bytes 2026-02-22T21:51:13.015 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16200 IOs, 6621023232 bytes 2026-02-22T21:51:13.059 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16300 IOs, 6646329344 bytes 2026-02-22T21:51:13.120 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16400 IOs, 6717762560 bytes 2026-02-22T21:51:13.162 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16500 IOs, 6742479872 bytes 2026-02-22T21:51:13.218 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16600 IOs, 6804902912 bytes 2026-02-22T21:51:13.266 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16700 IOs, 6829208576 bytes 2026-02-22T21:51:13.314 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16800 IOs, 6862182400 bytes 2026-02-22T21:51:13.356 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16900 IOs, 6890002432 bytes 2026-02-22T21:51:13.405 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17000 IOs, 6924876800 bytes 2026-02-22T21:51:13.459 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17100 IOs, 6967643136 bytes 2026-02-22T21:51:13.521 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17200 IOs, 7019244544 bytes 2026-02-22T21:51:13.579 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17300 IOs, 7098105856 bytes 2026-02-22T21:51:13.621 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17400 IOs, 7104052224 bytes 2026-02-22T21:51:13.668 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17500 IOs, 7136928768 bytes 2026-02-22T21:51:13.711 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17600 IOs, 7170800640 bytes 2026-02-22T21:51:13.752 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17700 IOs, 7206410240 bytes 2026-02-22T21:51:13.804 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17800 IOs, 7241057280 bytes 2026-02-22T21:51:13.849 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17900 IOs, 7278712832 bytes 2026-02-22T21:51:13.923 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18000 IOs, 7372326912 bytes 2026-02-22T21:51:13.958 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18100 IOs, 7387701248 bytes 2026-02-22T21:51:13.996 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18200 IOs, 7399600128 bytes 2026-02-22T21:51:14.033 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18300 IOs, 7412643840 bytes 2026-02-22T21:51:14.075 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18400 IOs, 7440668672 bytes 2026-02-22T21:51:14.150 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18500 IOs, 7519004672 bytes 2026-02-22T21:51:14.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18600 IOs, 7541960704 bytes 2026-02-22T21:51:14.223 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18700 IOs, 7543817216 bytes 2026-02-22T21:51:14.254 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18800 IOs, 7549561856 bytes 2026-02-22T21:51:14.302 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18900 IOs, 7578177536 bytes 2026-02-22T21:51:14.359 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19000 IOs, 7644561408 bytes 2026-02-22T21:51:14.426 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19100 IOs, 7705672704 bytes 2026-02-22T21:51:14.466 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19200 IOs, 7719252992 bytes 2026-02-22T21:51:14.520 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19300 IOs, 7758487552 bytes 2026-02-22T21:51:14.565 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19400 IOs, 7790571520 bytes 2026-02-22T21:51:14.639 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19500 IOs, 7872590848 bytes 2026-02-22T21:51:14.696 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19600 IOs, 7931142144 bytes 2026-02-22T21:51:14.771 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19700 IOs, 8035405824 bytes 2026-02-22T21:51:14.841 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19800 IOs, 8125939712 bytes 2026-02-22T21:51:14.887 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19900 IOs, 8146039808 bytes 2026-02-22T21:51:14.928 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20000 IOs, 8171450368 bytes 2026-02-22T21:51:14.958 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20100 IOs, 8172958720 bytes 2026-02-22T21:51:15.021 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20200 IOs, 8253561856 bytes 2026-02-22T21:51:15.071 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20300 IOs, 8300812288 bytes 2026-02-22T21:51:15.117 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20400 IOs, 8326602752 bytes 2026-02-22T21:51:15.163 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ error_code=124 2026-02-22T21:51:15.163 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set -e 2026-02-22T21:51:15.163 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' 124 -eq 124 ']' 2026-02-22T21:51:15.163 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:51:15.163 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T21:51:15.163 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_20 2026-02-22T21:51:15.163 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image_and_enable_mirror cluster2 mirror image_20 snapshot 128M 2026-02-22T21:51:15.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:51:15.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:51:15.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:51:15.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:51:15.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_20 2026-02-22T21:51:15.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:51:15.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T21:51:15.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n snapshot ']' 2026-02-22T21:51:15.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:51:15.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image cluster2 mirror image_20 128M 2026-02-22T21:51:15.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:51:15.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:51:15.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:51:15.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:51:15.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_20 2026-02-22T21:51:15.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:51:15.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local size=128 2026-02-22T21:51:15.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n 128M ']' 2026-02-22T21:51:15.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ size=128M 2026-02-22T21:51:15.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:51:15.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_20 2026-02-22T21:51:15.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T21:51:15.195 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:51:15.195 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' image = image ']' 2026-02-22T21:51:15.195 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:51:15.195 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ enable_mirror cluster2 mirror image_20 snapshot 2026-02-22T21:51:15.195 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:51:15.195 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:51:15.195 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_20 2026-02-22T21:51:15.195 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T21:51:15.195 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_20 snapshot 2026-02-22T21:51:16.135 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Mirroring enabled 2026-02-22T21:51:16.139 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 info mirror/image_20 2026-02-22T21:51:16.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_20': 2026-02-22T21:51:16.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 128 MiB in 32 objects 2026-02-22T21:51:16.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T21:51:16.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 1 2026-02-22T21:51:16.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 134633ea6de8 2026-02-22T21:51:16.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.134633ea6de8 2026-02-22T21:51:16.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T21:51:16.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock 2026-02-22T21:51:16.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T21:51:16.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T21:51:16.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:51:15 2026 2026-02-22T21:51:16.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:51:15 2026 2026-02-22T21:51:16.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:51:15 2026 2026-02-22T21:51:16.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T21:51:16.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T21:51:16.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: 50d0516c-a4f2-4b0b-9b37-15ee9b73d3ad 2026-02-22T21:51:16.166 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: true 2026-02-22T21:51:16.166 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n '' ']' 2026-02-22T21:51:16.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ stress_write_image cluster2 mirror image_20 2026-02-22T21:51:16.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:51:16.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:51:16.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_20 2026-02-22T21:51:16.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-22T21:51:16.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local duration=6 2026-02-22T21:51:16.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set +e 2026-02-22T21:51:16.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ timeout 6s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_20 --debug-rbd=20 --debug-journaler=20 2026-02-22T21:51:16.456 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:100 IOs, 63348736 bytes 2026-02-22T21:51:16.507 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:200 IOs, 104941568 bytes 2026-02-22T21:51:16.546 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:300 IOs, 141039616 bytes 2026-02-22T21:51:16.600 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:400 IOs, 193467392 bytes 2026-02-22T21:51:16.640 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:500 IOs, 227509248 bytes 2026-02-22T21:51:16.691 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:600 IOs, 271029248 bytes 2026-02-22T21:51:16.732 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:700 IOs, 294225920 bytes 2026-02-22T21:51:16.771 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:800 IOs, 315764736 bytes 2026-02-22T21:51:16.829 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:900 IOs, 377110528 bytes 2026-02-22T21:51:16.918 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1000 IOs, 510853120 bytes 2026-02-22T21:51:16.962 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1100 IOs, 543224832 bytes 2026-02-22T21:51:17.021 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1200 IOs, 609362944 bytes 2026-02-22T21:51:17.062 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1300 IOs, 635098112 bytes 2026-02-22T21:51:17.110 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1400 IOs, 671026176 bytes 2026-02-22T21:51:17.155 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1500 IOs, 702010368 bytes 2026-02-22T21:51:17.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1600 IOs, 754060288 bytes 2026-02-22T21:51:17.214 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.0 2026-02-22T21:51:17.215 INFO:teuthology.orchestra.run:waiting for 600 2026-02-22T21:51:17.215 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.0 2026-02-22T21:51:17.215 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Restarting daemon 2026-02-22T21:51:17.215 DEBUG:teuthology.orchestra.run.trial026:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 2026-02-22T21:51:17.218 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Started 2026-02-22T21:51:17.218 INFO:tasks.rbd_mirror_thrash:waiting for 116.9547999910121 secs before thrashing 2026-02-22T21:51:17.266 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1700 IOs, 843049984 bytes 2026-02-22T21:51:17.357 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1800 IOs, 953236480 bytes 2026-02-22T21:51:17.412 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1900 IOs, 1012605952 bytes 2026-02-22T21:51:17.449 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2000 IOs, 1031319552 bytes 2026-02-22T21:51:17.492 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2100 IOs, 1060468736 bytes 2026-02-22T21:51:17.528 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2200 IOs, 1075983360 bytes 2026-02-22T21:51:17.588 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2300 IOs, 1155227648 bytes 2026-02-22T21:51:17.643 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2400 IOs, 1220588544 bytes 2026-02-22T21:51:17.679 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2500 IOs, 1228035072 bytes 2026-02-22T21:51:17.715 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2600 IOs, 1249508352 bytes 2026-02-22T21:51:17.770 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2700 IOs, 1310143488 bytes 2026-02-22T21:51:17.815 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2800 IOs, 1340283904 bytes 2026-02-22T21:51:17.862 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2900 IOs, 1401022464 bytes 2026-02-22T21:51:17.929 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3000 IOs, 1476345856 bytes 2026-02-22T21:51:17.981 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3100 IOs, 1531872256 bytes 2026-02-22T21:51:18.029 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3200 IOs, 1579216896 bytes 2026-02-22T21:51:18.082 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3300 IOs, 1637690368 bytes 2026-02-22T21:51:18.130 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3400 IOs, 1668451328 bytes 2026-02-22T21:51:18.175 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3500 IOs, 1701997568 bytes 2026-02-22T21:51:18.250 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3600 IOs, 1802662912 bytes 2026-02-22T21:51:18.293 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3700 IOs, 1831130112 bytes 2026-02-22T21:51:18.325 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3800 IOs, 1841357824 bytes 2026-02-22T21:51:18.377 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3900 IOs, 1893226496 bytes 2026-02-22T21:51:18.421 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4000 IOs, 1913824256 bytes 2026-02-22T21:51:18.474 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4100 IOs, 1966767104 bytes 2026-02-22T21:51:18.529 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4200 IOs, 2022578176 bytes 2026-02-22T21:51:18.572 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4300 IOs, 2053872640 bytes 2026-02-22T21:51:18.650 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4400 IOs, 2162910208 bytes 2026-02-22T21:51:18.684 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4500 IOs, 2173702144 bytes 2026-02-22T21:51:18.747 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4600 IOs, 2257000448 bytes 2026-02-22T21:51:18.786 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4700 IOs, 2290859008 bytes 2026-02-22T21:51:18.850 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4800 IOs, 2357180416 bytes 2026-02-22T21:51:18.904 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4900 IOs, 2415580160 bytes 2026-02-22T21:51:18.950 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5000 IOs, 2440431616 bytes 2026-02-22T21:51:18.988 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5100 IOs, 2455217152 bytes 2026-02-22T21:51:19.029 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5200 IOs, 2477341696 bytes 2026-02-22T21:51:19.078 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5300 IOs, 2532878336 bytes 2026-02-22T21:51:19.112 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5400 IOs, 2544079872 bytes 2026-02-22T21:51:19.170 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5500 IOs, 2601406464 bytes 2026-02-22T21:51:19.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5600 IOs, 2622659584 bytes 2026-02-22T21:51:19.245 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5700 IOs, 2647614464 bytes 2026-02-22T21:51:19.279 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5800 IOs, 2661428224 bytes 2026-02-22T21:51:19.325 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5900 IOs, 2721269760 bytes 2026-02-22T21:51:19.392 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6000 IOs, 2769627136 bytes 2026-02-22T21:51:19.420 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6100 IOs, 2770221056 bytes 2026-02-22T21:51:19.468 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6200 IOs, 2800880640 bytes 2026-02-22T21:51:19.529 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6300 IOs, 2864690176 bytes 2026-02-22T21:51:19.570 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6400 IOs, 2888542208 bytes 2026-02-22T21:51:19.632 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6500 IOs, 2972730368 bytes 2026-02-22T21:51:19.688 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6600 IOs, 3038837760 bytes 2026-02-22T21:51:19.736 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6700 IOs, 3070356480 bytes 2026-02-22T21:51:19.792 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6800 IOs, 3134522368 bytes 2026-02-22T21:51:19.831 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6900 IOs, 3163240448 bytes 2026-02-22T21:51:19.878 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7000 IOs, 3201089536 bytes 2026-02-22T21:51:19.931 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7100 IOs, 3243891712 bytes 2026-02-22T21:51:19.966 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7200 IOs, 3271250944 bytes 2026-02-22T21:51:20.043 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7300 IOs, 3368467456 bytes 2026-02-22T21:51:20.080 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7400 IOs, 3381020672 bytes 2026-02-22T21:51:20.121 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7500 IOs, 3404163072 bytes 2026-02-22T21:51:20.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7600 IOs, 3464065024 bytes 2026-02-22T21:51:20.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7700 IOs, 3475437568 bytes 2026-02-22T21:51:20.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7800 IOs, 3581118464 bytes 2026-02-22T21:51:20.335 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7900 IOs, 3642899456 bytes 2026-02-22T21:51:20.407 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8000 IOs, 3730079744 bytes 2026-02-22T21:51:20.453 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8100 IOs, 3763723264 bytes 2026-02-22T21:51:20.487 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8200 IOs, 3782731776 bytes 2026-02-22T21:51:20.519 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8300 IOs, 3789080576 bytes 2026-02-22T21:51:20.575 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8400 IOs, 3854444544 bytes 2026-02-22T21:51:20.618 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8500 IOs, 3875620864 bytes 2026-02-22T21:51:20.696 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8600 IOs, 3978112000 bytes 2026-02-22T21:51:20.742 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8700 IOs, 4012684288 bytes 2026-02-22T21:51:20.784 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8800 IOs, 4045908992 bytes 2026-02-22T21:51:20.818 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8900 IOs, 4052661248 bytes 2026-02-22T21:51:20.862 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9000 IOs, 4087355392 bytes 2026-02-22T21:51:20.901 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9100 IOs, 4102798336 bytes 2026-02-22T21:51:20.930 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9200 IOs, 4103435264 bytes 2026-02-22T21:51:21.009 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9300 IOs, 4212711424 bytes 2026-02-22T21:51:21.065 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9400 IOs, 4273132544 bytes 2026-02-22T21:51:21.133 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9500 IOs, 4353141760 bytes 2026-02-22T21:51:21.183 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9600 IOs, 4414468096 bytes 2026-02-22T21:51:21.248 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9700 IOs, 4503370752 bytes 2026-02-22T21:51:21.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9800 IOs, 4503975936 bytes 2026-02-22T21:51:21.317 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9900 IOs, 4535782400 bytes 2026-02-22T21:51:21.388 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10000 IOs, 4623193088 bytes 2026-02-22T21:51:21.446 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10100 IOs, 4689395712 bytes 2026-02-22T21:51:21.487 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10200 IOs, 4718072832 bytes 2026-02-22T21:51:21.522 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10300 IOs, 4727251968 bytes 2026-02-22T21:51:21.572 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10400 IOs, 4767208448 bytes 2026-02-22T21:51:21.603 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10500 IOs, 4769876992 bytes 2026-02-22T21:51:21.643 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10600 IOs, 4816056320 bytes 2026-02-22T21:51:21.697 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10700 IOs, 4845894656 bytes 2026-02-22T21:51:21.750 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10800 IOs, 4900418560 bytes 2026-02-22T21:51:21.806 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10900 IOs, 4959059968 bytes 2026-02-22T21:51:21.837 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11000 IOs, 4969543680 bytes 2026-02-22T21:51:21.877 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11100 IOs, 4992287744 bytes 2026-02-22T21:51:21.913 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11200 IOs, 5005611008 bytes 2026-02-22T21:51:21.965 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11300 IOs, 5064926208 bytes 2026-02-22T21:51:22.010 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11400 IOs, 5096237056 bytes 2026-02-22T21:51:22.045 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11500 IOs, 5114977280 bytes 2026-02-22T21:51:22.104 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11600 IOs, 5192685568 bytes 2026-02-22T21:51:22.154 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11700 IOs, 5229059072 bytes 2026-02-22T21:51:22.174 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ error_code=124 2026-02-22T21:51:22.174 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set -e 2026-02-22T21:51:22.174 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' 124 -eq 124 ']' 2026-02-22T21:51:22.174 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:51:22.174 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T21:51:22.175 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_21 2026-02-22T21:51:22.175 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image_and_enable_mirror cluster2 mirror image_21 snapshot 128M 2026-02-22T21:51:22.175 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:51:22.175 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:51:22.175 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:51:22.175 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:51:22.175 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_21 2026-02-22T21:51:22.175 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:51:22.175 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T21:51:22.175 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n snapshot ']' 2026-02-22T21:51:22.175 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:51:22.175 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image cluster2 mirror image_21 128M 2026-02-22T21:51:22.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:51:22.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:51:22.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:51:22.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:51:22.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_21 2026-02-22T21:51:22.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:51:22.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local size=128 2026-02-22T21:51:22.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n 128M ']' 2026-02-22T21:51:22.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ size=128M 2026-02-22T21:51:22.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:51:22.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_21 2026-02-22T21:51:22.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T21:51:22.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:51:22.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' image = image ']' 2026-02-22T21:51:22.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:51:22.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ enable_mirror cluster2 mirror image_21 snapshot 2026-02-22T21:51:22.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:51:22.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:51:22.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_21 2026-02-22T21:51:22.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T21:51:22.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_21 snapshot 2026-02-22T21:51:23.143 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Mirroring enabled 2026-02-22T21:51:23.148 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 info mirror/image_21 2026-02-22T21:51:23.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_21': 2026-02-22T21:51:23.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 128 MiB in 32 objects 2026-02-22T21:51:23.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T21:51:23.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 1 2026-02-22T21:51:23.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 135ab0d3e185 2026-02-22T21:51:23.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.135ab0d3e185 2026-02-22T21:51:23.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T21:51:23.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock 2026-02-22T21:51:23.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T21:51:23.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T21:51:23.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:51:22 2026 2026-02-22T21:51:23.173 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:51:22 2026 2026-02-22T21:51:23.173 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:51:22 2026 2026-02-22T21:51:23.173 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T21:51:23.173 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T21:51:23.173 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: fcfc28ec-1e7e-4846-a752-e36745d49c74 2026-02-22T21:51:23.173 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: true 2026-02-22T21:51:23.174 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n '' ']' 2026-02-22T21:51:23.174 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ stress_write_image cluster2 mirror image_21 2026-02-22T21:51:23.174 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:51:23.174 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:51:23.174 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_21 2026-02-22T21:51:23.174 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-22T21:51:23.175 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local duration=5 2026-02-22T21:51:23.175 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set +e 2026-02-22T21:51:23.175 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ timeout 5s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_21 --debug-rbd=20 --debug-journaler=20 2026-02-22T21:51:23.256 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:100 IOs, 37040128 bytes 2026-02-22T21:51:23.298 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:200 IOs, 74030080 bytes 2026-02-22T21:51:23.345 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:300 IOs, 118715392 bytes 2026-02-22T21:51:23.401 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:400 IOs, 188010496 bytes 2026-02-22T21:51:23.655 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:500 IOs, 242800640 bytes 2026-02-22T21:51:23.689 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:600 IOs, 286082048 bytes 2026-02-22T21:51:23.746 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:700 IOs, 324551680 bytes 2026-02-22T21:51:23.798 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:800 IOs, 371538944 bytes 2026-02-22T21:51:23.854 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:900 IOs, 437283840 bytes 2026-02-22T21:51:23.887 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1000 IOs, 449829888 bytes 2026-02-22T21:51:23.936 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1100 IOs, 498248704 bytes 2026-02-22T21:51:23.981 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1200 IOs, 529417216 bytes 2026-02-22T21:51:24.027 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1300 IOs, 570790912 bytes 2026-02-22T21:51:24.071 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1400 IOs, 606159872 bytes 2026-02-22T21:51:24.106 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1500 IOs, 621975552 bytes 2026-02-22T21:51:24.141 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1600 IOs, 633348096 bytes 2026-02-22T21:51:24.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1700 IOs, 659397632 bytes 2026-02-22T21:51:24.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1800 IOs, 673243136 bytes 2026-02-22T21:51:24.273 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1900 IOs, 727673856 bytes 2026-02-22T21:51:24.308 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2000 IOs, 745695232 bytes 2026-02-22T21:51:24.376 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2100 IOs, 842465280 bytes 2026-02-22T21:51:24.426 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2200 IOs, 865449984 bytes 2026-02-22T21:51:24.472 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2300 IOs, 905243648 bytes 2026-02-22T21:51:24.516 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2400 IOs, 923767808 bytes 2026-02-22T21:51:24.557 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2500 IOs, 945595392 bytes 2026-02-22T21:51:24.602 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2600 IOs, 970497024 bytes 2026-02-22T21:51:24.640 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2700 IOs, 995622912 bytes 2026-02-22T21:51:24.674 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2800 IOs, 1012506624 bytes 2026-02-22T21:51:24.727 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2900 IOs, 1072492544 bytes 2026-02-22T21:51:24.769 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3000 IOs, 1085491200 bytes 2026-02-22T21:51:24.819 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3100 IOs, 1129224192 bytes 2026-02-22T21:51:24.866 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3200 IOs, 1168039936 bytes 2026-02-22T21:51:24.926 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3300 IOs, 1231557632 bytes 2026-02-22T21:51:24.985 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3400 IOs, 1291162624 bytes 2026-02-22T21:51:25.034 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3500 IOs, 1335887872 bytes 2026-02-22T21:51:25.066 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3600 IOs, 1347061760 bytes 2026-02-22T21:51:25.128 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3700 IOs, 1418626048 bytes 2026-02-22T21:51:25.159 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3800 IOs, 1437708288 bytes 2026-02-22T21:51:25.224 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3900 IOs, 1504107520 bytes 2026-02-22T21:51:25.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4000 IOs, 1576056832 bytes 2026-02-22T21:51:25.329 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4100 IOs, 1612467200 bytes 2026-02-22T21:51:25.359 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4200 IOs, 1616180224 bytes 2026-02-22T21:51:25.390 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4300 IOs, 1625029632 bytes 2026-02-22T21:51:25.428 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4400 IOs, 1647262720 bytes 2026-02-22T21:51:25.467 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4500 IOs, 1671958528 bytes 2026-02-22T21:51:25.499 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4600 IOs, 1679561728 bytes 2026-02-22T21:51:25.555 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4700 IOs, 1724226560 bytes 2026-02-22T21:51:25.603 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4800 IOs, 1773726720 bytes 2026-02-22T21:51:25.647 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4900 IOs, 1791630336 bytes 2026-02-22T21:51:25.683 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5000 IOs, 1812195328 bytes 2026-02-22T21:51:25.712 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5100 IOs, 1815133184 bytes 2026-02-22T21:51:25.739 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5200 IOs, 1815757824 bytes 2026-02-22T21:51:25.791 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5300 IOs, 1859846144 bytes 2026-02-22T21:51:25.849 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5400 IOs, 1928213504 bytes 2026-02-22T21:51:25.886 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5500 IOs, 1970316288 bytes 2026-02-22T21:51:25.948 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5600 IOs, 2017093632 bytes 2026-02-22T21:51:26.003 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5700 IOs, 2067063808 bytes 2026-02-22T21:51:26.048 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5800 IOs, 2103781376 bytes 2026-02-22T21:51:26.097 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5900 IOs, 2144604160 bytes 2026-02-22T21:51:26.129 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6000 IOs, 2145190912 bytes 2026-02-22T21:51:26.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6100 IOs, 2204888064 bytes 2026-02-22T21:51:26.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6200 IOs, 2253264896 bytes 2026-02-22T21:51:26.283 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6300 IOs, 2301426688 bytes 2026-02-22T21:51:26.329 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6400 IOs, 2333677568 bytes 2026-02-22T21:51:26.365 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6500 IOs, 2350397440 bytes 2026-02-22T21:51:26.417 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6600 IOs, 2408281088 bytes 2026-02-22T21:51:26.477 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6700 IOs, 2456788992 bytes 2026-02-22T21:51:26.512 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6800 IOs, 2462469120 bytes 2026-02-22T21:51:26.554 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6900 IOs, 2495548416 bytes 2026-02-22T21:51:26.598 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7000 IOs, 2547778560 bytes 2026-02-22T21:51:26.651 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7100 IOs, 2570229760 bytes 2026-02-22T21:51:26.686 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7200 IOs, 2579693568 bytes 2026-02-22T21:51:26.724 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7300 IOs, 2611432448 bytes 2026-02-22T21:51:26.759 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7400 IOs, 2612068352 bytes 2026-02-22T21:51:26.823 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7500 IOs, 2680711168 bytes 2026-02-22T21:51:26.890 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7600 IOs, 2771506176 bytes 2026-02-22T21:51:26.956 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7700 IOs, 2871766016 bytes 2026-02-22T21:51:27.022 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7800 IOs, 2914366464 bytes 2026-02-22T21:51:27.116 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7900 IOs, 3067023360 bytes 2026-02-22T21:51:27.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8000 IOs, 3102526464 bytes 2026-02-22T21:51:27.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8100 IOs, 3154231296 bytes 2026-02-22T21:51:27.274 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8200 IOs, 3230342144 bytes 2026-02-22T21:51:27.334 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8300 IOs, 3298298880 bytes 2026-02-22T21:51:27.363 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8400 IOs, 3298906112 bytes 2026-02-22T21:51:27.425 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8500 IOs, 3375766528 bytes 2026-02-22T21:51:27.460 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8600 IOs, 3385845760 bytes 2026-02-22T21:51:27.489 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8700 IOs, 3398800384 bytes 2026-02-22T21:51:27.542 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8800 IOs, 3422368768 bytes 2026-02-22T21:51:27.602 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8900 IOs, 3498309632 bytes 2026-02-22T21:51:27.652 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9000 IOs, 3526646784 bytes 2026-02-22T21:51:27.684 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9100 IOs, 3536007168 bytes 2026-02-22T21:51:27.720 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9200 IOs, 3562899456 bytes 2026-02-22T21:51:27.756 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9300 IOs, 3573524480 bytes 2026-02-22T21:51:27.793 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9400 IOs, 3592898560 bytes 2026-02-22T21:51:27.838 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9500 IOs, 3652733952 bytes 2026-02-22T21:51:27.885 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9600 IOs, 3686832128 bytes 2026-02-22T21:51:27.933 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9700 IOs, 3723984896 bytes 2026-02-22T21:51:27.970 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9800 IOs, 3742202880 bytes 2026-02-22T21:51:28.014 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9900 IOs, 3762567168 bytes 2026-02-22T21:51:28.046 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10000 IOs, 3772828672 bytes 2026-02-22T21:51:28.095 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10100 IOs, 3810667520 bytes 2026-02-22T21:51:28.157 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10200 IOs, 3884589056 bytes 2026-02-22T21:51:28.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ error_code=124 2026-02-22T21:51:28.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set -e 2026-02-22T21:51:28.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' 124 -eq 124 ']' 2026-02-22T21:51:28.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:51:28.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T21:51:28.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_22 2026-02-22T21:51:28.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image_and_enable_mirror cluster2 mirror image_22 snapshot 128M 2026-02-22T21:51:28.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:51:28.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:51:28.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:51:28.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:51:28.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_22 2026-02-22T21:51:28.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:51:28.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T21:51:28.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n snapshot ']' 2026-02-22T21:51:28.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:51:28.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image cluster2 mirror image_22 128M 2026-02-22T21:51:28.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:51:28.183 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:51:28.183 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:51:28.183 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:51:28.183 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_22 2026-02-22T21:51:28.183 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:51:28.183 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local size=128 2026-02-22T21:51:28.183 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n 128M ']' 2026-02-22T21:51:28.183 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ size=128M 2026-02-22T21:51:28.183 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:51:28.183 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_22 2026-02-22T21:51:28.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T21:51:28.213 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:51:28.213 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' image = image ']' 2026-02-22T21:51:28.213 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:51:28.213 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ enable_mirror cluster2 mirror image_22 snapshot 2026-02-22T21:51:28.213 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:51:28.213 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:51:28.213 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_22 2026-02-22T21:51:28.213 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T21:51:28.213 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_22 snapshot 2026-02-22T21:51:29.148 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Mirroring enabled 2026-02-22T21:51:29.153 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 info mirror/image_22 2026-02-22T21:51:29.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_22': 2026-02-22T21:51:29.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 128 MiB in 32 objects 2026-02-22T21:51:29.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T21:51:29.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 1 2026-02-22T21:51:29.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 136252b014b2 2026-02-22T21:51:29.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.136252b014b2 2026-02-22T21:51:29.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T21:51:29.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock 2026-02-22T21:51:29.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T21:51:29.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T21:51:29.180 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:51:28 2026 2026-02-22T21:51:29.180 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:51:28 2026 2026-02-22T21:51:29.180 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:51:28 2026 2026-02-22T21:51:29.180 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T21:51:29.180 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T21:51:29.180 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: cbed6911-da61-47b0-a3cc-2ddeb6616c4c 2026-02-22T21:51:29.180 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: true 2026-02-22T21:51:29.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n '' ']' 2026-02-22T21:51:29.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ stress_write_image cluster2 mirror image_22 2026-02-22T21:51:29.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:51:29.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:51:29.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_22 2026-02-22T21:51:29.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-22T21:51:29.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local duration=14 2026-02-22T21:51:29.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set +e 2026-02-22T21:51:29.183 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ timeout 14s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_22 --debug-rbd=20 --debug-journaler=20 2026-02-22T21:51:29.269 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:100 IOs, 47260672 bytes 2026-02-22T21:51:29.319 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:200 IOs, 91851776 bytes 2026-02-22T21:51:29.371 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:300 IOs, 142865408 bytes 2026-02-22T21:51:29.420 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:400 IOs, 177916928 bytes 2026-02-22T21:51:29.471 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:500 IOs, 234890240 bytes 2026-02-22T21:51:29.499 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:600 IOs, 235500544 bytes 2026-02-22T21:51:29.551 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:700 IOs, 273840128 bytes 2026-02-22T21:51:29.613 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:800 IOs, 343951360 bytes 2026-02-22T21:51:29.665 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:900 IOs, 392291328 bytes 2026-02-22T21:51:29.709 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1000 IOs, 426059776 bytes 2026-02-22T21:51:29.735 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1100 IOs, 426662912 bytes 2026-02-22T21:51:29.775 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1200 IOs, 456717312 bytes 2026-02-22T21:51:29.855 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1300 IOs, 554421248 bytes 2026-02-22T21:51:29.896 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1400 IOs, 579250176 bytes 2026-02-22T21:51:29.962 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1500 IOs, 674953216 bytes 2026-02-22T21:51:30.017 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1600 IOs, 725139456 bytes 2026-02-22T21:51:30.063 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1700 IOs, 754398208 bytes 2026-02-22T21:51:30.141 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1800 IOs, 868952064 bytes 2026-02-22T21:51:30.195 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1900 IOs, 903654400 bytes 2026-02-22T21:51:30.225 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2000 IOs, 920445952 bytes 2026-02-22T21:51:30.265 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2100 IOs, 934926336 bytes 2026-02-22T21:51:30.322 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2200 IOs, 1031729152 bytes 2026-02-22T21:51:30.407 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2300 IOs, 1112704000 bytes 2026-02-22T21:51:30.434 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2400 IOs, 1113320448 bytes 2026-02-22T21:51:30.471 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2500 IOs, 1138935808 bytes 2026-02-22T21:51:30.519 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2600 IOs, 1179761664 bytes 2026-02-22T21:51:30.562 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2700 IOs, 1196765184 bytes 2026-02-22T21:51:30.616 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2800 IOs, 1203264512 bytes 2026-02-22T21:51:30.659 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2900 IOs, 1233989632 bytes 2026-02-22T21:51:30.729 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3000 IOs, 1301729280 bytes 2026-02-22T21:51:30.765 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3100 IOs, 1313291264 bytes 2026-02-22T21:51:30.810 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3200 IOs, 1357585408 bytes 2026-02-22T21:51:30.854 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3300 IOs, 1376726016 bytes 2026-02-22T21:51:30.897 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3400 IOs, 1400370176 bytes 2026-02-22T21:51:30.959 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3500 IOs, 1450028032 bytes 2026-02-22T21:51:30.990 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3600 IOs, 1450657792 bytes 2026-02-22T21:51:31.026 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3700 IOs, 1470611456 bytes 2026-02-22T21:51:31.065 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3800 IOs, 1491976192 bytes 2026-02-22T21:51:31.108 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3900 IOs, 1522316288 bytes 2026-02-22T21:51:31.170 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4000 IOs, 1593463808 bytes 2026-02-22T21:51:31.214 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4100 IOs, 1629388800 bytes 2026-02-22T21:51:31.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4200 IOs, 1685570560 bytes 2026-02-22T21:51:31.318 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4300 IOs, 1725135872 bytes 2026-02-22T21:51:31.390 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4400 IOs, 1817441280 bytes 2026-02-22T21:51:31.465 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4500 IOs, 1915817984 bytes 2026-02-22T21:51:31.499 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4600 IOs, 1948768256 bytes 2026-02-22T21:51:31.557 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4700 IOs, 1990920192 bytes 2026-02-22T21:51:31.606 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4800 IOs, 2012219392 bytes 2026-02-22T21:51:31.665 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4900 IOs, 2063796224 bytes 2026-02-22T21:51:31.711 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5000 IOs, 2094393344 bytes 2026-02-22T21:51:31.756 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5100 IOs, 2126156800 bytes 2026-02-22T21:51:31.791 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5200 IOs, 2155595776 bytes 2026-02-22T21:51:31.892 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5300 IOs, 2273721344 bytes 2026-02-22T21:51:31.938 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5400 IOs, 2320802816 bytes 2026-02-22T21:51:31.991 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5500 IOs, 2360065024 bytes 2026-02-22T21:51:32.038 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5600 IOs, 2403439616 bytes 2026-02-22T21:51:32.083 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5700 IOs, 2442596352 bytes 2026-02-22T21:51:32.127 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5800 IOs, 2475248640 bytes 2026-02-22T21:51:32.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5900 IOs, 2521563136 bytes 2026-02-22T21:51:32.236 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6000 IOs, 2587816960 bytes 2026-02-22T21:51:32.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6100 IOs, 2608492544 bytes 2026-02-22T21:51:32.328 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6200 IOs, 2656478208 bytes 2026-02-22T21:51:32.355 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6300 IOs, 2658483200 bytes 2026-02-22T21:51:32.393 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6400 IOs, 2677179392 bytes 2026-02-22T21:51:32.442 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6500 IOs, 2714265600 bytes 2026-02-22T21:51:32.509 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6600 IOs, 2785551360 bytes 2026-02-22T21:51:32.572 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6700 IOs, 2847532032 bytes 2026-02-22T21:51:32.633 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6800 IOs, 2882838528 bytes 2026-02-22T21:51:32.696 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6900 IOs, 2943351808 bytes 2026-02-22T21:51:32.751 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7000 IOs, 2997975040 bytes 2026-02-22T21:51:32.801 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7100 IOs, 3027613696 bytes 2026-02-22T21:51:32.835 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7200 IOs, 3036774400 bytes 2026-02-22T21:51:32.882 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7300 IOs, 3089709056 bytes 2026-02-22T21:51:32.935 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7400 IOs, 3122480128 bytes 2026-02-22T21:51:33.010 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7500 IOs, 3187314688 bytes 2026-02-22T21:51:33.062 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7600 IOs, 3243538432 bytes 2026-02-22T21:51:33.111 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7700 IOs, 3281270784 bytes 2026-02-22T21:51:33.159 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7800 IOs, 3324533760 bytes 2026-02-22T21:51:33.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7900 IOs, 3368777728 bytes 2026-02-22T21:51:33.235 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8000 IOs, 3369385984 bytes 2026-02-22T21:51:33.286 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8100 IOs, 3418024960 bytes 2026-02-22T21:51:33.340 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8200 IOs, 3473850368 bytes 2026-02-22T21:51:33.386 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8300 IOs, 3515828224 bytes 2026-02-22T21:51:33.464 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8400 IOs, 3645119488 bytes 2026-02-22T21:51:33.536 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8500 IOs, 3731431424 bytes 2026-02-22T21:51:33.593 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8600 IOs, 3765825536 bytes 2026-02-22T21:51:33.659 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8700 IOs, 3831215104 bytes 2026-02-22T21:51:33.717 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8800 IOs, 3893479424 bytes 2026-02-22T21:51:33.778 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8900 IOs, 3966631936 bytes 2026-02-22T21:51:33.832 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9000 IOs, 4020421632 bytes 2026-02-22T21:51:33.893 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9100 IOs, 4079819776 bytes 2026-02-22T21:51:33.973 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9200 IOs, 4181619712 bytes 2026-02-22T21:51:34.005 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9300 IOs, 4182224896 bytes 2026-02-22T21:51:34.048 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9400 IOs, 4211662848 bytes 2026-02-22T21:51:34.109 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9500 IOs, 4275092480 bytes 2026-02-22T21:51:34.150 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9600 IOs, 4294935552 bytes 2026-02-22T21:51:34.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9700 IOs, 4316091392 bytes 2026-02-22T21:51:34.225 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9800 IOs, 4340081664 bytes 2026-02-22T21:51:34.256 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9900 IOs, 4352685056 bytes 2026-02-22T21:51:34.299 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10000 IOs, 4368998400 bytes 2026-02-22T21:51:34.330 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10100 IOs, 4376750080 bytes 2026-02-22T21:51:34.390 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10200 IOs, 4439031808 bytes 2026-02-22T21:51:34.460 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10300 IOs, 4533147648 bytes 2026-02-22T21:51:34.507 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10400 IOs, 4568073216 bytes 2026-02-22T21:51:34.543 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10500 IOs, 4584643584 bytes 2026-02-22T21:51:34.615 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10600 IOs, 4660231168 bytes 2026-02-22T21:51:34.661 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10700 IOs, 4683906048 bytes 2026-02-22T21:51:34.690 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10800 IOs, 4684526592 bytes 2026-02-22T21:51:34.732 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10900 IOs, 4718097408 bytes 2026-02-22T21:51:34.799 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11000 IOs, 4789787648 bytes 2026-02-22T21:51:34.858 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11100 IOs, 4831178752 bytes 2026-02-22T21:51:34.910 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11200 IOs, 4858406912 bytes 2026-02-22T21:51:34.957 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11300 IOs, 4905754624 bytes 2026-02-22T21:51:35.012 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11400 IOs, 4948042752 bytes 2026-02-22T21:51:35.056 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11500 IOs, 4981340160 bytes 2026-02-22T21:51:35.123 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11600 IOs, 5066855424 bytes 2026-02-22T21:51:35.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11700 IOs, 5103086592 bytes 2026-02-22T21:51:35.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11800 IOs, 5187440640 bytes 2026-02-22T21:51:35.276 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11900 IOs, 5201888256 bytes 2026-02-22T21:51:35.325 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12000 IOs, 5262337024 bytes 2026-02-22T21:51:35.393 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12100 IOs, 5327411200 bytes 2026-02-22T21:51:35.464 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12200 IOs, 5421016064 bytes 2026-02-22T21:51:35.509 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12300 IOs, 5446798336 bytes 2026-02-22T21:51:35.541 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12400 IOs, 5456769024 bytes 2026-02-22T21:51:35.585 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12500 IOs, 5485303808 bytes 2026-02-22T21:51:35.625 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12600 IOs, 5509959680 bytes 2026-02-22T21:51:35.676 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12700 IOs, 5565868032 bytes 2026-02-22T21:51:35.733 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12800 IOs, 5593912320 bytes 2026-02-22T21:51:35.806 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12900 IOs, 5688469504 bytes 2026-02-22T21:51:35.859 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13000 IOs, 5733484544 bytes 2026-02-22T21:51:35.911 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13100 IOs, 5786972160 bytes 2026-02-22T21:51:35.937 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13200 IOs, 5787601920 bytes 2026-02-22T21:51:35.996 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13300 IOs, 5859708928 bytes 2026-02-22T21:51:36.052 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13400 IOs, 5910050816 bytes 2026-02-22T21:51:36.103 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13500 IOs, 5954701312 bytes 2026-02-22T21:51:36.153 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13600 IOs, 6002899968 bytes 2026-02-22T21:51:36.195 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13700 IOs, 6029050880 bytes 2026-02-22T21:51:36.230 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13800 IOs, 6043188224 bytes 2026-02-22T21:51:36.267 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13900 IOs, 6070882304 bytes 2026-02-22T21:51:36.336 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14000 IOs, 6146333696 bytes 2026-02-22T21:51:36.367 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14100 IOs, 6146926592 bytes 2026-02-22T21:51:36.421 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14200 IOs, 6208037888 bytes 2026-02-22T21:51:36.482 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14300 IOs, 6272896000 bytes 2026-02-22T21:51:36.521 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14400 IOs, 6273524736 bytes 2026-02-22T21:51:36.561 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14500 IOs, 6296368128 bytes 2026-02-22T21:51:36.609 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14600 IOs, 6339542016 bytes 2026-02-22T21:51:36.655 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14700 IOs, 6366424064 bytes 2026-02-22T21:51:36.714 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14800 IOs, 6425000960 bytes 2026-02-22T21:51:36.753 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14900 IOs, 6444974080 bytes 2026-02-22T21:51:36.801 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15000 IOs, 6484827136 bytes 2026-02-22T21:51:36.840 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15100 IOs, 6496839680 bytes 2026-02-22T21:51:36.891 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15200 IOs, 6552579072 bytes 2026-02-22T21:51:36.928 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15300 IOs, 6580124672 bytes 2026-02-22T21:51:36.991 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15400 IOs, 6631784448 bytes 2026-02-22T21:51:37.056 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15500 IOs, 6698590208 bytes 2026-02-22T21:51:37.101 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15600 IOs, 6740287488 bytes 2026-02-22T21:51:37.143 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15700 IOs, 6776161280 bytes 2026-02-22T21:51:37.183 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15800 IOs, 6785744896 bytes 2026-02-22T21:51:37.222 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15900 IOs, 6833719296 bytes 2026-02-22T21:51:37.281 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16000 IOs, 6879846400 bytes 2026-02-22T21:51:37.357 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16100 IOs, 6943333376 bytes 2026-02-22T21:51:37.392 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16200 IOs, 6969968640 bytes 2026-02-22T21:51:37.450 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16300 IOs, 7020988416 bytes 2026-02-22T21:51:37.520 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16400 IOs, 7108156416 bytes 2026-02-22T21:51:37.574 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16500 IOs, 7157352448 bytes 2026-02-22T21:51:37.611 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16600 IOs, 7167172608 bytes 2026-02-22T21:51:37.654 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16700 IOs, 7194492928 bytes 2026-02-22T21:51:37.691 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16800 IOs, 7201994752 bytes 2026-02-22T21:51:37.727 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16900 IOs, 7226506240 bytes 2026-02-22T21:51:37.761 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17000 IOs, 7246689280 bytes 2026-02-22T21:51:37.825 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17100 IOs, 7298766848 bytes 2026-02-22T21:51:37.865 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17200 IOs, 7325671424 bytes 2026-02-22T21:51:37.909 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17300 IOs, 7359472640 bytes 2026-02-22T21:51:37.951 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17400 IOs, 7380372480 bytes 2026-02-22T21:51:38.014 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17500 IOs, 7450867712 bytes 2026-02-22T21:51:38.052 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17600 IOs, 7472520192 bytes 2026-02-22T21:51:38.095 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17700 IOs, 7512982528 bytes 2026-02-22T21:51:38.150 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17800 IOs, 7568358400 bytes 2026-02-22T21:51:38.185 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17900 IOs, 7570173952 bytes 2026-02-22T21:51:38.237 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18000 IOs, 7621559296 bytes 2026-02-22T21:51:38.293 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18100 IOs, 7685099520 bytes 2026-02-22T21:51:38.345 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18200 IOs, 7721880576 bytes 2026-02-22T21:51:38.399 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18300 IOs, 7765811200 bytes 2026-02-22T21:51:38.462 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18400 IOs, 7845680128 bytes 2026-02-22T21:51:38.517 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18500 IOs, 7884926976 bytes 2026-02-22T21:51:38.562 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18600 IOs, 7920999424 bytes 2026-02-22T21:51:38.603 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18700 IOs, 7969230848 bytes 2026-02-22T21:51:38.657 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18800 IOs, 8004680704 bytes 2026-02-22T21:51:38.715 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18900 IOs, 8059291648 bytes 2026-02-22T21:51:38.766 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19000 IOs, 8102969344 bytes 2026-02-22T21:51:38.811 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19100 IOs, 8130648064 bytes 2026-02-22T21:51:38.858 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19200 IOs, 8163833856 bytes 2026-02-22T21:51:38.886 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19300 IOs, 8170971136 bytes 2026-02-22T21:51:38.941 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19400 IOs, 8225935360 bytes 2026-02-22T21:51:38.988 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19500 IOs, 8269762560 bytes 2026-02-22T21:51:39.048 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19600 IOs, 8325211136 bytes 2026-02-22T21:51:39.111 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19700 IOs, 8396922880 bytes 2026-02-22T21:51:39.162 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19800 IOs, 8432710656 bytes 2026-02-22T21:51:39.188 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19900 IOs, 8433310720 bytes 2026-02-22T21:51:39.220 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20000 IOs, 8440832000 bytes 2026-02-22T21:51:39.249 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20100 IOs, 8461247488 bytes 2026-02-22T21:51:39.306 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20200 IOs, 8507487232 bytes 2026-02-22T21:51:39.350 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20300 IOs, 8540567552 bytes 2026-02-22T21:51:39.390 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20400 IOs, 8566112256 bytes 2026-02-22T21:51:39.427 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20500 IOs, 8605081600 bytes 2026-02-22T21:51:39.481 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20600 IOs, 8634455040 bytes 2026-02-22T21:51:39.521 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20700 IOs, 8656239616 bytes 2026-02-22T21:51:39.568 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20800 IOs, 8693541888 bytes 2026-02-22T21:51:39.600 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20900 IOs, 8703280128 bytes 2026-02-22T21:51:39.659 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21000 IOs, 8770331648 bytes 2026-02-22T21:51:39.734 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21100 IOs, 8866823168 bytes 2026-02-22T21:51:39.801 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21200 IOs, 8935846912 bytes 2026-02-22T21:51:39.864 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21300 IOs, 8992711680 bytes 2026-02-22T21:51:39.920 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21400 IOs, 9047924736 bytes 2026-02-22T21:51:39.957 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21500 IOs, 9064817664 bytes 2026-02-22T21:51:40.011 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21600 IOs, 9119099904 bytes 2026-02-22T21:51:40.090 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21700 IOs, 9245494272 bytes 2026-02-22T21:51:40.146 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21800 IOs, 9308171264 bytes 2026-02-22T21:51:40.184 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21900 IOs, 9308800000 bytes 2026-02-22T21:51:40.236 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22000 IOs, 9364759552 bytes 2026-02-22T21:51:40.263 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22100 IOs, 9388103680 bytes 2026-02-22T21:51:40.299 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22200 IOs, 9409868800 bytes 2026-02-22T21:51:40.368 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22300 IOs, 9482658816 bytes 2026-02-22T21:51:40.429 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22400 IOs, 9538724864 bytes 2026-02-22T21:51:40.483 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22500 IOs, 9584700416 bytes 2026-02-22T21:51:40.531 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22600 IOs, 9622824960 bytes 2026-02-22T21:51:40.579 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22700 IOs, 9647129600 bytes 2026-02-22T21:51:40.609 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22800 IOs, 9649617920 bytes 2026-02-22T21:51:40.669 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22900 IOs, 9707194368 bytes 2026-02-22T21:51:40.699 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23000 IOs, 9708587008 bytes 2026-02-22T21:51:40.766 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23100 IOs, 9775131648 bytes 2026-02-22T21:51:40.831 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23200 IOs, 9835994112 bytes 2026-02-22T21:51:40.876 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23300 IOs, 9875814400 bytes 2026-02-22T21:51:40.931 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23400 IOs, 9935171584 bytes 2026-02-22T21:51:40.983 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23500 IOs, 9978319872 bytes 2026-02-22T21:51:41.027 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23600 IOs, 10004640768 bytes 2026-02-22T21:51:41.082 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23700 IOs, 10061153280 bytes 2026-02-22T21:51:41.116 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23800 IOs, 10079214592 bytes 2026-02-22T21:51:41.170 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23900 IOs, 10147648512 bytes 2026-02-22T21:51:41.218 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24000 IOs, 10168551424 bytes 2026-02-22T21:51:41.254 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24100 IOs, 10183816192 bytes 2026-02-22T21:51:41.310 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24200 IOs, 10231728128 bytes 2026-02-22T21:51:41.354 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24300 IOs, 10268064768 bytes 2026-02-22T21:51:41.403 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24400 IOs, 10305545216 bytes 2026-02-22T21:51:41.439 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24500 IOs, 10322707456 bytes 2026-02-22T21:51:41.482 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24600 IOs, 10353200128 bytes 2026-02-22T21:51:41.528 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24700 IOs, 10384518144 bytes 2026-02-22T21:51:41.575 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24800 IOs, 10415546368 bytes 2026-02-22T21:51:41.617 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24900 IOs, 10438493184 bytes 2026-02-22T21:51:41.660 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25000 IOs, 10454550528 bytes 2026-02-22T21:51:41.700 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25100 IOs, 10473364480 bytes 2026-02-22T21:51:41.763 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25200 IOs, 10543398912 bytes 2026-02-22T21:51:41.798 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25300 IOs, 10551593984 bytes 2026-02-22T21:51:41.845 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25400 IOs, 10612348928 bytes 2026-02-22T21:51:41.908 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25500 IOs, 10647778304 bytes 2026-02-22T21:51:41.967 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25600 IOs, 10716362752 bytes 2026-02-22T21:51:42.004 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25700 IOs, 10737469440 bytes 2026-02-22T21:51:42.082 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25800 IOs, 10843174912 bytes 2026-02-22T21:51:42.125 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25900 IOs, 10862735360 bytes 2026-02-22T21:51:42.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:26000 IOs, 10890912768 bytes 2026-02-22T21:51:42.225 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:26100 IOs, 10964066304 bytes 2026-02-22T21:51:42.294 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:26200 IOs, 11047317504 bytes 2026-02-22T21:51:42.358 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:26300 IOs, 11108327424 bytes 2026-02-22T21:51:42.411 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:26400 IOs, 11162891264 bytes 2026-02-22T21:51:42.456 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:26500 IOs, 11211209728 bytes 2026-02-22T21:51:42.501 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:26600 IOs, 11226263552 bytes 2026-02-22T21:51:42.549 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:26700 IOs, 11268704256 bytes 2026-02-22T21:51:42.615 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:26800 IOs, 11337415680 bytes 2026-02-22T21:51:42.654 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:26900 IOs, 11374257152 bytes 2026-02-22T21:51:42.750 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:27000 IOs, 11471875072 bytes 2026-02-22T21:51:42.845 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:27100 IOs, 11626678272 bytes 2026-02-22T21:51:42.904 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:27200 IOs, 11681383424 bytes 2026-02-22T21:51:42.937 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:27300 IOs, 11687772160 bytes 2026-02-22T21:51:42.974 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:27400 IOs, 11705801728 bytes 2026-02-22T21:51:43.026 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:27500 IOs, 11752847360 bytes 2026-02-22T21:51:43.071 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:27600 IOs, 11808016384 bytes 2026-02-22T21:51:43.145 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:27700 IOs, 11858150400 bytes 2026-02-22T21:51:43.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:27800 IOs, 11894409216 bytes 2026-02-22T21:51:43.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ error_code=124 2026-02-22T21:51:43.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set -e 2026-02-22T21:51:43.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' 124 -eq 124 ']' 2026-02-22T21:51:43.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:51:43.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T21:51:43.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_23 2026-02-22T21:51:43.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image_and_enable_mirror cluster2 mirror image_23 snapshot 128M 2026-02-22T21:51:43.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:51:43.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:51:43.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:51:43.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:51:43.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_23 2026-02-22T21:51:43.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:51:43.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T21:51:43.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n snapshot ']' 2026-02-22T21:51:43.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:51:43.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image cluster2 mirror image_23 128M 2026-02-22T21:51:43.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:51:43.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:51:43.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:51:43.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:51:43.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_23 2026-02-22T21:51:43.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:51:43.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local size=128 2026-02-22T21:51:43.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n 128M ']' 2026-02-22T21:51:43.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ size=128M 2026-02-22T21:51:43.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:51:43.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_23 2026-02-22T21:51:43.220 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T21:51:43.220 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:51:43.220 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' image = image ']' 2026-02-22T21:51:43.220 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:51:43.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ enable_mirror cluster2 mirror image_23 snapshot 2026-02-22T21:51:43.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:51:43.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:51:43.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_23 2026-02-22T21:51:43.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T21:51:43.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_23 snapshot 2026-02-22T21:51:44.159 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Mirroring enabled 2026-02-22T21:51:44.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 info mirror/image_23 2026-02-22T21:51:44.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_23': 2026-02-22T21:51:44.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 128 MiB in 32 objects 2026-02-22T21:51:44.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T21:51:44.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 1 2026-02-22T21:51:44.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 136bafd4c542 2026-02-22T21:51:44.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.136bafd4c542 2026-02-22T21:51:44.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T21:51:44.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock 2026-02-22T21:51:44.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T21:51:44.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T21:51:44.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:51:43 2026 2026-02-22T21:51:44.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:51:43 2026 2026-02-22T21:51:44.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:51:43 2026 2026-02-22T21:51:44.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T21:51:44.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T21:51:44.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: 707a358f-f62b-47ff-a4e3-ae016127a25e 2026-02-22T21:51:44.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: true 2026-02-22T21:51:44.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n '' ']' 2026-02-22T21:51:44.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ stress_write_image cluster2 mirror image_23 2026-02-22T21:51:44.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:51:44.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:51:44.195 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_23 2026-02-22T21:51:44.195 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-22T21:51:44.195 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local duration=11 2026-02-22T21:51:44.195 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set +e 2026-02-22T21:51:44.195 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ timeout 11s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_23 --debug-rbd=20 --debug-journaler=20 2026-02-22T21:51:44.289 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:100 IOs, 70493184 bytes 2026-02-22T21:51:44.333 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:200 IOs, 116344832 bytes 2026-02-22T21:51:44.403 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:300 IOs, 212499456 bytes 2026-02-22T21:51:44.460 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:400 IOs, 268581888 bytes 2026-02-22T21:51:44.501 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:500 IOs, 301018112 bytes 2026-02-22T21:51:44.543 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:600 IOs, 338408448 bytes 2026-02-22T21:51:44.572 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:700 IOs, 339008512 bytes 2026-02-22T21:51:44.620 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:800 IOs, 392039424 bytes 2026-02-22T21:51:44.653 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:900 IOs, 404622336 bytes 2026-02-22T21:51:44.692 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1000 IOs, 451773440 bytes 2026-02-22T21:51:44.759 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1100 IOs, 516624384 bytes 2026-02-22T21:51:44.794 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1200 IOs, 534659072 bytes 2026-02-22T21:51:44.831 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1300 IOs, 547824640 bytes 2026-02-22T21:51:44.880 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1400 IOs, 594010112 bytes 2026-02-22T21:51:44.916 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1500 IOs, 616783872 bytes 2026-02-22T21:51:44.981 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1600 IOs, 716505088 bytes 2026-02-22T21:51:45.051 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1700 IOs, 803687424 bytes 2026-02-22T21:51:45.098 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1800 IOs, 833026048 bytes 2026-02-22T21:51:45.138 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1900 IOs, 860761088 bytes 2026-02-22T21:51:45.174 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2000 IOs, 866797568 bytes 2026-02-22T21:51:45.222 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2100 IOs, 918077440 bytes 2026-02-22T21:51:45.259 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2200 IOs, 938596352 bytes 2026-02-22T21:51:45.300 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2300 IOs, 966310912 bytes 2026-02-22T21:51:45.336 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2400 IOs, 983458816 bytes 2026-02-22T21:51:45.378 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2500 IOs, 1011624960 bytes 2026-02-22T21:51:45.434 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2600 IOs, 1084414976 bytes 2026-02-22T21:51:45.491 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2700 IOs, 1148285952 bytes 2026-02-22T21:51:45.540 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2800 IOs, 1198296064 bytes 2026-02-22T21:51:45.601 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2900 IOs, 1275048960 bytes 2026-02-22T21:51:45.645 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3000 IOs, 1322180608 bytes 2026-02-22T21:51:45.699 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3100 IOs, 1395885056 bytes 2026-02-22T21:51:45.747 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3200 IOs, 1420139520 bytes 2026-02-22T21:51:45.801 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3300 IOs, 1461497856 bytes 2026-02-22T21:51:45.849 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3400 IOs, 1497129984 bytes 2026-02-22T21:51:45.886 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3500 IOs, 1521267712 bytes 2026-02-22T21:51:45.922 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3600 IOs, 1539592192 bytes 2026-02-22T21:51:45.953 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3700 IOs, 1542841344 bytes 2026-02-22T21:51:45.994 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3800 IOs, 1569096704 bytes 2026-02-22T21:51:46.056 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3900 IOs, 1634723840 bytes 2026-02-22T21:51:46.111 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4000 IOs, 1676442624 bytes 2026-02-22T21:51:46.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4100 IOs, 1752625152 bytes 2026-02-22T21:51:46.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4200 IOs, 1760515072 bytes 2026-02-22T21:51:46.286 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4300 IOs, 1869927424 bytes 2026-02-22T21:51:46.339 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4400 IOs, 1900132352 bytes 2026-02-22T21:51:46.392 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4500 IOs, 1953106944 bytes 2026-02-22T21:51:46.461 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4600 IOs, 2038112256 bytes 2026-02-22T21:51:46.503 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4700 IOs, 2058902528 bytes 2026-02-22T21:51:46.553 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4800 IOs, 2106792960 bytes 2026-02-22T21:51:46.600 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4900 IOs, 2142980096 bytes 2026-02-22T21:51:46.631 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5000 IOs, 2143603712 bytes 2026-02-22T21:51:46.672 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5100 IOs, 2180447232 bytes 2026-02-22T21:51:46.725 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5200 IOs, 2225957888 bytes 2026-02-22T21:51:46.786 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5300 IOs, 2280334336 bytes 2026-02-22T21:51:46.844 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5400 IOs, 2348715008 bytes 2026-02-22T21:51:46.883 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5500 IOs, 2384599040 bytes 2026-02-22T21:51:46.978 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5600 IOs, 2526609408 bytes 2026-02-22T21:51:47.037 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5700 IOs, 2607906816 bytes 2026-02-22T21:51:47.083 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5800 IOs, 2617279488 bytes 2026-02-22T21:51:47.161 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5900 IOs, 2711840768 bytes 2026-02-22T21:51:47.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6000 IOs, 2753497088 bytes 2026-02-22T21:51:47.264 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6100 IOs, 2818224128 bytes 2026-02-22T21:51:47.333 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6200 IOs, 2889791488 bytes 2026-02-22T21:51:47.372 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6300 IOs, 2903526400 bytes 2026-02-22T21:51:47.415 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6400 IOs, 2925252608 bytes 2026-02-22T21:51:47.457 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6500 IOs, 2962171904 bytes 2026-02-22T21:51:47.501 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6600 IOs, 2991456256 bytes 2026-02-22T21:51:47.540 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6700 IOs, 3011469312 bytes 2026-02-22T21:51:47.579 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6800 IOs, 3036707840 bytes 2026-02-22T21:51:47.630 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6900 IOs, 3086480384 bytes 2026-02-22T21:51:47.713 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7000 IOs, 3199090688 bytes 2026-02-22T21:51:47.762 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7100 IOs, 3242130432 bytes 2026-02-22T21:51:47.812 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7200 IOs, 3297912832 bytes 2026-02-22T21:51:47.852 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7300 IOs, 3314547712 bytes 2026-02-22T21:51:47.891 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7400 IOs, 3334550528 bytes 2026-02-22T21:51:47.939 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7500 IOs, 3403317248 bytes 2026-02-22T21:51:47.994 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7600 IOs, 3443057664 bytes 2026-02-22T21:51:48.056 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7700 IOs, 3510690816 bytes 2026-02-22T21:51:48.108 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7800 IOs, 3559546880 bytes 2026-02-22T21:51:48.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7900 IOs, 3618809856 bytes 2026-02-22T21:51:48.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8000 IOs, 3653789696 bytes 2026-02-22T21:51:48.251 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8100 IOs, 3693431808 bytes 2026-02-22T21:51:48.301 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8200 IOs, 3738334208 bytes 2026-02-22T21:51:48.351 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8300 IOs, 3779351552 bytes 2026-02-22T21:51:48.402 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8400 IOs, 3835668480 bytes 2026-02-22T21:51:48.460 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8500 IOs, 3897415680 bytes 2026-02-22T21:51:48.510 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8600 IOs, 3939270656 bytes 2026-02-22T21:51:48.549 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8700 IOs, 3965256704 bytes 2026-02-22T21:51:48.586 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8800 IOs, 3989584896 bytes 2026-02-22T21:51:48.639 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8900 IOs, 4048884736 bytes 2026-02-22T21:51:48.688 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9000 IOs, 4093156352 bytes 2026-02-22T21:51:48.733 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9100 IOs, 4121765888 bytes 2026-02-22T21:51:48.790 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9200 IOs, 4183746560 bytes 2026-02-22T21:51:48.839 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9300 IOs, 4221232128 bytes 2026-02-22T21:51:48.887 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9400 IOs, 4268612608 bytes 2026-02-22T21:51:48.937 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9500 IOs, 4322999296 bytes 2026-02-22T21:51:48.981 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9600 IOs, 4357911552 bytes 2026-02-22T21:51:49.018 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9700 IOs, 4383232000 bytes 2026-02-22T21:51:49.062 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9800 IOs, 4418862080 bytes 2026-02-22T21:51:49.110 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9900 IOs, 4467141632 bytes 2026-02-22T21:51:49.166 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10000 IOs, 4518714368 bytes 2026-02-22T21:51:49.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10100 IOs, 4542576640 bytes 2026-02-22T21:51:49.265 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10200 IOs, 4599409664 bytes 2026-02-22T21:51:49.308 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10300 IOs, 4639949824 bytes 2026-02-22T21:51:49.350 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10400 IOs, 4663929856 bytes 2026-02-22T21:51:49.401 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10500 IOs, 4720474112 bytes 2026-02-22T21:51:49.459 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10600 IOs, 4783134720 bytes 2026-02-22T21:51:49.505 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10700 IOs, 4832061440 bytes 2026-02-22T21:51:49.541 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10800 IOs, 4846101504 bytes 2026-02-22T21:51:49.607 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10900 IOs, 4928681984 bytes 2026-02-22T21:51:49.662 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11000 IOs, 4979622912 bytes 2026-02-22T21:51:49.720 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11100 IOs, 5031418880 bytes 2026-02-22T21:51:49.759 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11200 IOs, 5055263744 bytes 2026-02-22T21:51:49.819 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11300 IOs, 5113517056 bytes 2026-02-22T21:51:49.881 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11400 IOs, 5202862080 bytes 2026-02-22T21:51:49.957 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11500 IOs, 5287363584 bytes 2026-02-22T21:51:49.985 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11600 IOs, 5287956480 bytes 2026-02-22T21:51:50.022 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11700 IOs, 5313911808 bytes 2026-02-22T21:51:50.065 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11800 IOs, 5349822464 bytes 2026-02-22T21:51:50.133 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11900 IOs, 5409025024 bytes 2026-02-22T21:51:50.188 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12000 IOs, 5473597440 bytes 2026-02-22T21:51:50.234 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12100 IOs, 5506551808 bytes 2026-02-22T21:51:50.265 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12200 IOs, 5519892480 bytes 2026-02-22T21:51:50.306 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12300 IOs, 5557894144 bytes 2026-02-22T21:51:50.349 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12400 IOs, 5576088576 bytes 2026-02-22T21:51:50.391 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12500 IOs, 5603108864 bytes 2026-02-22T21:51:50.449 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12600 IOs, 5658868736 bytes 2026-02-22T21:51:50.510 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12700 IOs, 5732957184 bytes 2026-02-22T21:51:50.537 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12800 IOs, 5733563392 bytes 2026-02-22T21:51:50.573 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12900 IOs, 5757170688 bytes 2026-02-22T21:51:50.621 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13000 IOs, 5808416768 bytes 2026-02-22T21:51:50.677 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13100 IOs, 5857476608 bytes 2026-02-22T21:51:50.731 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13200 IOs, 5920162816 bytes 2026-02-22T21:51:50.782 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13300 IOs, 5967140864 bytes 2026-02-22T21:51:50.833 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13400 IOs, 5990381568 bytes 2026-02-22T21:51:50.880 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13500 IOs, 6033933312 bytes 2026-02-22T21:51:50.930 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13600 IOs, 6078765056 bytes 2026-02-22T21:51:50.985 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13700 IOs, 6140924928 bytes 2026-02-22T21:51:51.031 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13800 IOs, 6172969984 bytes 2026-02-22T21:51:51.084 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13900 IOs, 6271515648 bytes 2026-02-22T21:51:51.153 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14000 IOs, 6330809344 bytes 2026-02-22T21:51:51.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14100 IOs, 6376927232 bytes 2026-02-22T21:51:51.245 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14200 IOs, 6398755840 bytes 2026-02-22T21:51:51.285 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14300 IOs, 6425795584 bytes 2026-02-22T21:51:51.315 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14400 IOs, 6426387456 bytes 2026-02-22T21:51:51.348 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14500 IOs, 6437720064 bytes 2026-02-22T21:51:51.395 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14600 IOs, 6496308224 bytes 2026-02-22T21:51:51.466 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14700 IOs, 6606573568 bytes 2026-02-22T21:51:51.524 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14800 IOs, 6652414976 bytes 2026-02-22T21:51:51.589 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14900 IOs, 6745531392 bytes 2026-02-22T21:51:51.629 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15000 IOs, 6771441664 bytes 2026-02-22T21:51:51.679 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15100 IOs, 6816391168 bytes 2026-02-22T21:51:51.724 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15200 IOs, 6861715456 bytes 2026-02-22T21:51:51.789 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15300 IOs, 6934738944 bytes 2026-02-22T21:51:51.842 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15400 IOs, 6971470848 bytes 2026-02-22T21:51:51.871 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15500 IOs, 6973350912 bytes 2026-02-22T21:51:51.909 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15600 IOs, 6996890624 bytes 2026-02-22T21:51:51.953 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15700 IOs, 7029130240 bytes 2026-02-22T21:51:51.990 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15800 IOs, 7043620864 bytes 2026-02-22T21:51:52.038 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15900 IOs, 7088999424 bytes 2026-02-22T21:51:52.076 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16000 IOs, 7100274688 bytes 2026-02-22T21:51:52.107 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16100 IOs, 7107024896 bytes 2026-02-22T21:51:52.133 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16200 IOs, 7107976192 bytes 2026-02-22T21:51:52.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16300 IOs, 7157904384 bytes 2026-02-22T21:51:52.230 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16400 IOs, 7193510912 bytes 2026-02-22T21:51:52.296 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16500 IOs, 7242313728 bytes 2026-02-22T21:51:52.348 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16600 IOs, 7291613184 bytes 2026-02-22T21:51:52.399 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16700 IOs, 7341073408 bytes 2026-02-22T21:51:52.437 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16800 IOs, 7360499712 bytes 2026-02-22T21:51:52.478 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16900 IOs, 7406904320 bytes 2026-02-22T21:51:52.541 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17000 IOs, 7467127808 bytes 2026-02-22T21:51:52.592 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17100 IOs, 7511078912 bytes 2026-02-22T21:51:52.632 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17200 IOs, 7525844992 bytes 2026-02-22T21:51:52.665 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17300 IOs, 7546789888 bytes 2026-02-22T21:51:52.711 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17400 IOs, 7583989760 bytes 2026-02-22T21:51:52.758 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17500 IOs, 7616257024 bytes 2026-02-22T21:51:52.811 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17600 IOs, 7669021696 bytes 2026-02-22T21:51:52.858 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17700 IOs, 7687160832 bytes 2026-02-22T21:51:52.904 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17800 IOs, 7727733760 bytes 2026-02-22T21:51:52.964 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17900 IOs, 7793036288 bytes 2026-02-22T21:51:53.004 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18000 IOs, 7827447808 bytes 2026-02-22T21:51:53.035 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18100 IOs, 7828073472 bytes 2026-02-22T21:51:53.079 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18200 IOs, 7862811648 bytes 2026-02-22T21:51:53.124 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18300 IOs, 7891081216 bytes 2026-02-22T21:51:53.169 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18400 IOs, 7926155264 bytes 2026-02-22T21:51:53.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18500 IOs, 7938425856 bytes 2026-02-22T21:51:53.267 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18600 IOs, 8002081792 bytes 2026-02-22T21:51:53.325 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18700 IOs, 8059722752 bytes 2026-02-22T21:51:53.370 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18800 IOs, 8100612096 bytes 2026-02-22T21:51:53.427 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18900 IOs, 8166125568 bytes 2026-02-22T21:51:53.494 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19000 IOs, 8247388160 bytes 2026-02-22T21:51:53.539 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19100 IOs, 8279987200 bytes 2026-02-22T21:51:53.570 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19200 IOs, 8285820928 bytes 2026-02-22T21:51:53.617 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19300 IOs, 8333905920 bytes 2026-02-22T21:51:53.660 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19400 IOs, 8358654976 bytes 2026-02-22T21:51:53.722 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19500 IOs, 8456116224 bytes 2026-02-22T21:51:53.777 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19600 IOs, 8499343360 bytes 2026-02-22T21:51:53.812 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19700 IOs, 8534927360 bytes 2026-02-22T21:51:53.859 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19800 IOs, 8549147648 bytes 2026-02-22T21:51:53.899 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19900 IOs, 8568156160 bytes 2026-02-22T21:51:53.970 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20000 IOs, 8661865472 bytes 2026-02-22T21:51:54.011 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20100 IOs, 8685770752 bytes 2026-02-22T21:51:54.054 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20200 IOs, 8713944064 bytes 2026-02-22T21:51:54.099 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20300 IOs, 8768856064 bytes 2026-02-22T21:51:54.137 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20400 IOs, 8769457152 bytes 2026-02-22T21:51:54.180 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20500 IOs, 8797057024 bytes 2026-02-22T21:51:54.241 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20600 IOs, 8859784192 bytes 2026-02-22T21:51:54.302 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20700 IOs, 8953430016 bytes 2026-02-22T21:51:54.350 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20800 IOs, 8992210944 bytes 2026-02-22T21:51:54.402 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20900 IOs, 9024691200 bytes 2026-02-22T21:51:54.446 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21000 IOs, 9057314816 bytes 2026-02-22T21:51:54.476 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21100 IOs, 9077571584 bytes 2026-02-22T21:51:54.511 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21200 IOs, 9078205440 bytes 2026-02-22T21:51:54.549 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21300 IOs, 9097251840 bytes 2026-02-22T21:51:54.618 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21400 IOs, 9190388736 bytes 2026-02-22T21:51:54.662 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21500 IOs, 9228801024 bytes 2026-02-22T21:51:54.710 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21600 IOs, 9270516736 bytes 2026-02-22T21:51:54.764 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21700 IOs, 9332528128 bytes 2026-02-22T21:51:54.816 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21800 IOs, 9372617728 bytes 2026-02-22T21:51:54.883 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21900 IOs, 9454512128 bytes 2026-02-22T21:51:54.935 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22000 IOs, 9501237248 bytes 2026-02-22T21:51:54.978 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22100 IOs, 9544214528 bytes 2026-02-22T21:51:55.041 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22200 IOs, 9606585344 bytes 2026-02-22T21:51:55.071 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22300 IOs, 9633809408 bytes 2026-02-22T21:51:55.129 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22400 IOs, 9672121344 bytes 2026-02-22T21:51:55.174 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22500 IOs, 9698097152 bytes 2026-02-22T21:51:55.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ error_code=124 2026-02-22T21:51:55.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set -e 2026-02-22T21:51:55.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' 124 -eq 124 ']' 2026-02-22T21:51:55.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:51:55.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T21:51:55.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_24 2026-02-22T21:51:55.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image_and_enable_mirror cluster2 mirror image_24 snapshot 128M 2026-02-22T21:51:55.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:51:55.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:51:55.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:51:55.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:51:55.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_24 2026-02-22T21:51:55.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:51:55.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T21:51:55.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n snapshot ']' 2026-02-22T21:51:55.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:51:55.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image cluster2 mirror image_24 128M 2026-02-22T21:51:55.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:51:55.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:51:55.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:51:55.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:51:55.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_24 2026-02-22T21:51:55.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:51:55.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local size=128 2026-02-22T21:51:55.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n 128M ']' 2026-02-22T21:51:55.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ size=128M 2026-02-22T21:51:55.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:51:55.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_24 2026-02-22T21:51:55.229 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T21:51:55.229 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:51:55.229 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' image = image ']' 2026-02-22T21:51:55.230 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:51:55.230 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ enable_mirror cluster2 mirror image_24 snapshot 2026-02-22T21:51:55.230 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:51:55.230 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:51:55.230 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_24 2026-02-22T21:51:55.230 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T21:51:55.230 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_24 snapshot 2026-02-22T21:51:56.219 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Mirroring enabled 2026-02-22T21:51:56.224 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 info mirror/image_24 2026-02-22T21:51:56.248 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_24': 2026-02-22T21:51:56.248 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 128 MiB in 32 objects 2026-02-22T21:51:56.248 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T21:51:56.248 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 1 2026-02-22T21:51:56.248 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 137467a99246 2026-02-22T21:51:56.248 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.137467a99246 2026-02-22T21:51:56.248 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T21:51:56.249 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock 2026-02-22T21:51:56.249 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T21:51:56.249 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T21:51:56.249 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:51:55 2026 2026-02-22T21:51:56.249 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:51:55 2026 2026-02-22T21:51:56.249 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:51:55 2026 2026-02-22T21:51:56.249 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T21:51:56.249 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T21:51:56.249 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: f06205bf-4d40-464a-9efe-4169410d86ad 2026-02-22T21:51:56.249 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: true 2026-02-22T21:51:56.250 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n '' ']' 2026-02-22T21:51:56.250 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ stress_write_image cluster2 mirror image_24 2026-02-22T21:51:56.250 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:51:56.251 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:51:56.251 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_24 2026-02-22T21:51:56.251 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-22T21:51:56.251 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local duration=8 2026-02-22T21:51:56.252 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set +e 2026-02-22T21:51:56.252 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ timeout 8s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_24 --debug-rbd=20 --debug-journaler=20 2026-02-22T21:51:56.338 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:100 IOs, 45680640 bytes 2026-02-22T21:51:56.410 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:200 IOs, 137523200 bytes 2026-02-22T21:51:56.478 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:300 IOs, 225577984 bytes 2026-02-22T21:51:56.520 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:400 IOs, 265613312 bytes 2026-02-22T21:51:56.571 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:500 IOs, 318145536 bytes 2026-02-22T21:51:56.611 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:600 IOs, 340528128 bytes 2026-02-22T21:51:56.644 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:700 IOs, 356868096 bytes 2026-02-22T21:51:56.706 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:800 IOs, 438761472 bytes 2026-02-22T21:51:56.763 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:900 IOs, 497491968 bytes 2026-02-22T21:51:56.821 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1000 IOs, 561225728 bytes 2026-02-22T21:51:56.874 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1100 IOs, 585622528 bytes 2026-02-22T21:51:56.913 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1200 IOs, 603649024 bytes 2026-02-22T21:51:56.942 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1300 IOs, 608628736 bytes 2026-02-22T21:51:56.982 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1400 IOs, 629986304 bytes 2026-02-22T21:51:57.040 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1500 IOs, 697747456 bytes 2026-02-22T21:51:57.095 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1600 IOs, 756615168 bytes 2026-02-22T21:51:57.147 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1700 IOs, 801452032 bytes 2026-02-22T21:51:57.185 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1800 IOs, 824207360 bytes 2026-02-22T21:51:57.231 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1900 IOs, 875871232 bytes 2026-02-22T21:51:57.281 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2000 IOs, 908765184 bytes 2026-02-22T21:51:57.327 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2100 IOs, 948248576 bytes 2026-02-22T21:51:57.375 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2200 IOs, 999115776 bytes 2026-02-22T21:51:57.431 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2300 IOs, 1044123648 bytes 2026-02-22T21:51:57.471 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2400 IOs, 1069044736 bytes 2026-02-22T21:51:57.520 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2500 IOs, 1149637632 bytes 2026-02-22T21:51:57.580 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2600 IOs, 1210457088 bytes 2026-02-22T21:51:57.638 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2700 IOs, 1261351936 bytes 2026-02-22T21:51:57.667 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2800 IOs, 1272743936 bytes 2026-02-22T21:51:57.728 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2900 IOs, 1328462848 bytes 2026-02-22T21:51:57.777 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3000 IOs, 1374866432 bytes 2026-02-22T21:51:57.815 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3100 IOs, 1395418112 bytes 2026-02-22T21:51:57.854 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3200 IOs, 1419691008 bytes 2026-02-22T21:51:57.907 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3300 IOs, 1460156416 bytes 2026-02-22T21:51:57.976 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3400 IOs, 1533540352 bytes 2026-02-22T21:51:58.033 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3500 IOs, 1604235264 bytes 2026-02-22T21:51:58.077 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3600 IOs, 1650197504 bytes 2026-02-22T21:51:58.118 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3700 IOs, 1658543104 bytes 2026-02-22T21:51:58.147 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3800 IOs, 1676662784 bytes 2026-02-22T21:51:58.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3900 IOs, 1754019840 bytes 2026-02-22T21:51:58.257 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4000 IOs, 1774575616 bytes 2026-02-22T21:51:58.290 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4100 IOs, 1792530432 bytes 2026-02-22T21:51:58.335 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4200 IOs, 1822418944 bytes 2026-02-22T21:51:58.382 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4300 IOs, 1856181248 bytes 2026-02-22T21:51:58.418 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4400 IOs, 1875350528 bytes 2026-02-22T21:51:58.456 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4500 IOs, 1901473792 bytes 2026-02-22T21:51:58.491 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4600 IOs, 1917378560 bytes 2026-02-22T21:51:58.546 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4700 IOs, 1972756480 bytes 2026-02-22T21:51:58.596 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4800 IOs, 2025246720 bytes 2026-02-22T21:51:58.641 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4900 IOs, 2050917376 bytes 2026-02-22T21:51:58.692 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5000 IOs, 2103672832 bytes 2026-02-22T21:51:58.738 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5100 IOs, 2138933248 bytes 2026-02-22T21:51:58.776 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5200 IOs, 2176862208 bytes 2026-02-22T21:51:58.841 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5300 IOs, 2226398208 bytes 2026-02-22T21:51:58.889 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5400 IOs, 2247271424 bytes 2026-02-22T21:51:58.945 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5500 IOs, 2325197824 bytes 2026-02-22T21:51:59.001 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5600 IOs, 2368413696 bytes 2026-02-22T21:51:59.062 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5700 IOs, 2429883392 bytes 2026-02-22T21:51:59.106 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5800 IOs, 2465206272 bytes 2026-02-22T21:51:59.157 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5900 IOs, 2501879808 bytes 2026-02-22T21:51:59.223 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6000 IOs, 2580321280 bytes 2026-02-22T21:51:59.264 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6100 IOs, 2611827712 bytes 2026-02-22T21:51:59.291 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6200 IOs, 2612442112 bytes 2026-02-22T21:51:59.352 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6300 IOs, 2683375616 bytes 2026-02-22T21:51:59.394 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6400 IOs, 2720074752 bytes 2026-02-22T21:51:59.445 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6500 IOs, 2764250112 bytes 2026-02-22T21:51:59.495 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6600 IOs, 2797503488 bytes 2026-02-22T21:51:59.566 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6700 IOs, 2888503296 bytes 2026-02-22T21:51:59.612 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6800 IOs, 2919190528 bytes 2026-02-22T21:51:59.656 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6900 IOs, 2957395968 bytes 2026-02-22T21:51:59.703 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7000 IOs, 2998553600 bytes 2026-02-22T21:51:59.765 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7100 IOs, 3073161216 bytes 2026-02-22T21:51:59.830 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7200 IOs, 3154021376 bytes 2026-02-22T21:51:59.912 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7300 IOs, 3256816640 bytes 2026-02-22T21:51:59.964 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7400 IOs, 3289721856 bytes 2026-02-22T21:52:00.013 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7500 IOs, 3330574336 bytes 2026-02-22T21:52:00.056 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7600 IOs, 3373892608 bytes 2026-02-22T21:52:00.116 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7700 IOs, 3415294976 bytes 2026-02-22T21:52:00.156 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7800 IOs, 3436391424 bytes 2026-02-22T21:52:00.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7900 IOs, 3493626880 bytes 2026-02-22T21:52:00.258 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8000 IOs, 3521106944 bytes 2026-02-22T21:52:00.307 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8100 IOs, 3584053248 bytes 2026-02-22T21:52:00.348 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8200 IOs, 3591826432 bytes 2026-02-22T21:52:00.402 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8300 IOs, 3642165248 bytes 2026-02-22T21:52:00.440 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8400 IOs, 3662649344 bytes 2026-02-22T21:52:00.492 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8500 IOs, 3715949568 bytes 2026-02-22T21:52:00.531 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8600 IOs, 3740519424 bytes 2026-02-22T21:52:00.573 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8700 IOs, 3758776320 bytes 2026-02-22T21:52:00.612 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8800 IOs, 3794232320 bytes 2026-02-22T21:52:00.662 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8900 IOs, 3829164032 bytes 2026-02-22T21:52:00.711 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9000 IOs, 3865903104 bytes 2026-02-22T21:52:00.739 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9100 IOs, 3866512384 bytes 2026-02-22T21:52:00.794 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9200 IOs, 3931560960 bytes 2026-02-22T21:52:00.846 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9300 IOs, 3969833984 bytes 2026-02-22T21:52:00.898 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9400 IOs, 4018249728 bytes 2026-02-22T21:52:00.964 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9500 IOs, 4096307200 bytes 2026-02-22T21:52:01.003 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9600 IOs, 4096900096 bytes 2026-02-22T21:52:01.063 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9700 IOs, 4154827776 bytes 2026-02-22T21:52:01.105 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9800 IOs, 4199272448 bytes 2026-02-22T21:52:01.143 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9900 IOs, 4203979776 bytes 2026-02-22T21:52:01.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10000 IOs, 4231904256 bytes 2026-02-22T21:52:01.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10100 IOs, 4238090240 bytes 2026-02-22T21:52:01.282 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10200 IOs, 4323195904 bytes 2026-02-22T21:52:01.317 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10300 IOs, 4337064960 bytes 2026-02-22T21:52:01.368 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10400 IOs, 4378964992 bytes 2026-02-22T21:52:01.394 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10500 IOs, 4379586560 bytes 2026-02-22T21:52:01.448 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10600 IOs, 4428672000 bytes 2026-02-22T21:52:01.496 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10700 IOs, 4476106752 bytes 2026-02-22T21:52:01.531 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10800 IOs, 4480835584 bytes 2026-02-22T21:52:01.571 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10900 IOs, 4528190464 bytes 2026-02-22T21:52:01.651 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11000 IOs, 4618943488 bytes 2026-02-22T21:52:01.702 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11100 IOs, 4668797952 bytes 2026-02-22T21:52:01.748 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11200 IOs, 4694260736 bytes 2026-02-22T21:52:01.803 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11300 IOs, 4759595008 bytes 2026-02-22T21:52:01.863 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11400 IOs, 4811020288 bytes 2026-02-22T21:52:01.916 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11500 IOs, 4871481344 bytes 2026-02-22T21:52:01.986 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11600 IOs, 4937726976 bytes 2026-02-22T21:52:02.023 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11700 IOs, 4955848704 bytes 2026-02-22T21:52:02.096 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11800 IOs, 5049656320 bytes 2026-02-22T21:52:02.157 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11900 IOs, 5118010368 bytes 2026-02-22T21:52:02.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12000 IOs, 5135781888 bytes 2026-02-22T21:52:02.246 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12100 IOs, 5182702592 bytes 2026-02-22T21:52:02.299 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12200 IOs, 5231737856 bytes 2026-02-22T21:52:02.358 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12300 IOs, 5307220992 bytes 2026-02-22T21:52:02.403 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12400 IOs, 5342120960 bytes 2026-02-22T21:52:02.453 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12500 IOs, 5401550848 bytes 2026-02-22T21:52:02.497 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12600 IOs, 5426595840 bytes 2026-02-22T21:52:02.530 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12700 IOs, 5431325696 bytes 2026-02-22T21:52:02.570 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12800 IOs, 5452578816 bytes 2026-02-22T21:52:02.628 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12900 IOs, 5517790208 bytes 2026-02-22T21:52:02.671 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13000 IOs, 5537253376 bytes 2026-02-22T21:52:02.729 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13100 IOs, 5628528640 bytes 2026-02-22T21:52:02.784 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13200 IOs, 5649204224 bytes 2026-02-22T21:52:02.836 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13300 IOs, 5715335168 bytes 2026-02-22T21:52:02.889 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13400 IOs, 5741071360 bytes 2026-02-22T21:52:02.945 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13500 IOs, 5786308608 bytes 2026-02-22T21:52:02.977 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13600 IOs, 5794460672 bytes 2026-02-22T21:52:03.019 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13700 IOs, 5819299840 bytes 2026-02-22T21:52:03.073 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13800 IOs, 5894704128 bytes 2026-02-22T21:52:03.111 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13900 IOs, 5908055040 bytes 2026-02-22T21:52:03.152 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14000 IOs, 5931681792 bytes 2026-02-22T21:52:03.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14100 IOs, 5969854464 bytes 2026-02-22T21:52:03.260 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14200 IOs, 6030396416 bytes 2026-02-22T21:52:03.308 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14300 IOs, 6090044416 bytes 2026-02-22T21:52:03.349 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14400 IOs, 6108207104 bytes 2026-02-22T21:52:03.425 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14500 IOs, 6218546176 bytes 2026-02-22T21:52:03.490 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14600 IOs, 6293552128 bytes 2026-02-22T21:52:03.557 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14700 IOs, 6357536768 bytes 2026-02-22T21:52:03.593 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14800 IOs, 6391124992 bytes 2026-02-22T21:52:03.630 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14900 IOs, 6394936320 bytes 2026-02-22T21:52:03.692 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15000 IOs, 6471223296 bytes 2026-02-22T21:52:03.724 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15100 IOs, 6479963136 bytes 2026-02-22T21:52:03.775 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15200 IOs, 6529445888 bytes 2026-02-22T21:52:03.846 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15300 IOs, 6594371584 bytes 2026-02-22T21:52:03.889 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15400 IOs, 6623058944 bytes 2026-02-22T21:52:03.924 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15500 IOs, 6637458432 bytes 2026-02-22T21:52:03.960 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15600 IOs, 6641887232 bytes 2026-02-22T21:52:03.994 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15700 IOs, 6662947840 bytes 2026-02-22T21:52:04.036 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15800 IOs, 6680224768 bytes 2026-02-22T21:52:04.070 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15900 IOs, 6695368704 bytes 2026-02-22T21:52:04.140 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16000 IOs, 6776666112 bytes 2026-02-22T21:52:04.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16100 IOs, 6808064000 bytes 2026-02-22T21:52:04.214 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16200 IOs, 6808679424 bytes 2026-02-22T21:52:04.246 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16300 IOs, 6810264576 bytes 2026-02-22T21:52:04.259 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ error_code=124 2026-02-22T21:52:04.259 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set -e 2026-02-22T21:52:04.259 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' 124 -eq 124 ']' 2026-02-22T21:52:04.259 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:52:04.260 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T21:52:04.260 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_25 2026-02-22T21:52:04.260 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image_and_enable_mirror cluster2 mirror image_25 snapshot 128M 2026-02-22T21:52:04.260 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:52:04.260 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:52:04.260 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:52:04.260 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:52:04.260 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_25 2026-02-22T21:52:04.260 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:52:04.260 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T21:52:04.260 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n snapshot ']' 2026-02-22T21:52:04.261 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:52:04.261 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image cluster2 mirror image_25 128M 2026-02-22T21:52:04.261 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:52:04.261 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:52:04.261 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:52:04.261 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:52:04.261 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_25 2026-02-22T21:52:04.261 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:52:04.262 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local size=128 2026-02-22T21:52:04.262 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n 128M ']' 2026-02-22T21:52:04.262 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ size=128M 2026-02-22T21:52:04.262 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:52:04.262 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_25 2026-02-22T21:52:04.289 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T21:52:04.289 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:52:04.289 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' image = image ']' 2026-02-22T21:52:04.289 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:52:04.289 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ enable_mirror cluster2 mirror image_25 snapshot 2026-02-22T21:52:04.289 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:52:04.289 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:52:04.289 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_25 2026-02-22T21:52:04.289 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T21:52:04.289 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_25 snapshot 2026-02-22T21:52:05.115 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Mirroring enabled 2026-02-22T21:52:05.120 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 info mirror/image_25 2026-02-22T21:52:05.145 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_25': 2026-02-22T21:52:05.145 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 128 MiB in 32 objects 2026-02-22T21:52:05.145 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T21:52:05.145 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 1 2026-02-22T21:52:05.145 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 137eb707aa78 2026-02-22T21:52:05.145 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.137eb707aa78 2026-02-22T21:52:05.146 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T21:52:05.146 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock 2026-02-22T21:52:05.146 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T21:52:05.146 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T21:52:05.146 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:52:04 2026 2026-02-22T21:52:05.146 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:52:04 2026 2026-02-22T21:52:05.146 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:52:04 2026 2026-02-22T21:52:05.146 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T21:52:05.146 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T21:52:05.146 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: 886eb68e-ef31-471e-9872-4fcf569230c4 2026-02-22T21:52:05.147 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: true 2026-02-22T21:52:05.147 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n '' ']' 2026-02-22T21:52:05.148 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ stress_write_image cluster2 mirror image_25 2026-02-22T21:52:05.148 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:52:05.148 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:52:05.148 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_25 2026-02-22T21:52:05.148 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-22T21:52:05.149 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local duration=11 2026-02-22T21:52:05.149 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set +e 2026-02-22T21:52:05.149 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ timeout 11s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_25 --debug-rbd=20 --debug-journaler=20 2026-02-22T21:52:05.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:100 IOs, 25198592 bytes 2026-02-22T21:52:05.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:200 IOs, 101997568 bytes 2026-02-22T21:52:05.338 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:300 IOs, 157116416 bytes 2026-02-22T21:52:05.375 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:400 IOs, 182587392 bytes 2026-02-22T21:52:05.427 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:500 IOs, 236980224 bytes 2026-02-22T21:52:05.465 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:600 IOs, 260346880 bytes 2026-02-22T21:52:05.527 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:700 IOs, 338607104 bytes 2026-02-22T21:52:05.569 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:800 IOs, 363524096 bytes 2026-02-22T21:52:05.609 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:900 IOs, 403096576 bytes 2026-02-22T21:52:05.657 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1000 IOs, 456777728 bytes 2026-02-22T21:52:05.718 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1100 IOs, 500344832 bytes 2026-02-22T21:52:05.746 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1200 IOs, 511019008 bytes 2026-02-22T21:52:05.774 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1300 IOs, 511613952 bytes 2026-02-22T21:52:05.817 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1400 IOs, 548455424 bytes 2026-02-22T21:52:05.892 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1500 IOs, 663632896 bytes 2026-02-22T21:52:05.939 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1600 IOs, 689346560 bytes 2026-02-22T21:52:06.010 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1700 IOs, 750587904 bytes 2026-02-22T21:52:06.054 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1800 IOs, 788992000 bytes 2026-02-22T21:52:06.102 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1900 IOs, 829312000 bytes 2026-02-22T21:52:06.138 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2000 IOs, 849425408 bytes 2026-02-22T21:52:06.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2100 IOs, 850046976 bytes 2026-02-22T21:52:06.247 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2200 IOs, 957319168 bytes 2026-02-22T21:52:06.296 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2300 IOs, 993873920 bytes 2026-02-22T21:52:06.334 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2400 IOs, 1009541120 bytes 2026-02-22T21:52:06.374 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2500 IOs, 1037880320 bytes 2026-02-22T21:52:06.427 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2600 IOs, 1095430144 bytes 2026-02-22T21:52:06.458 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2700 IOs, 1102036992 bytes 2026-02-22T21:52:06.519 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2800 IOs, 1184310272 bytes 2026-02-22T21:52:06.581 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2900 IOs, 1241812992 bytes 2026-02-22T21:52:06.664 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3000 IOs, 1366079488 bytes 2026-02-22T21:52:06.709 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3100 IOs, 1434894336 bytes 2026-02-22T21:52:06.782 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3200 IOs, 1509181440 bytes 2026-02-22T21:52:06.810 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3300 IOs, 1509827584 bytes 2026-02-22T21:52:06.848 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3400 IOs, 1544137728 bytes 2026-02-22T21:52:06.916 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3500 IOs, 1620096000 bytes 2026-02-22T21:52:06.934 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.5783744866947234) > weight (0.25) 2026-02-22T21:52:06.934 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.863073318673187) > weight (0.25) 2026-02-22T21:52:06.934 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.4364214593464988) > weight (0.25) 2026-02-22T21:52:06.934 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.3 with skip (0.5823318214593052) > weight (0.25) 2026-02-22T21:52:06.934 INFO:tasks.rbd_mirror_thrash:waiting for 71.54859742287968 secs before thrashing 2026-02-22T21:52:06.971 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3600 IOs, 1684252672 bytes 2026-02-22T21:52:07.015 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3700 IOs, 1697515520 bytes 2026-02-22T21:52:07.044 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3800 IOs, 1702490112 bytes 2026-02-22T21:52:07.094 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3900 IOs, 1746630656 bytes 2026-02-22T21:52:07.134 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4000 IOs, 1764076544 bytes 2026-02-22T21:52:07.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4100 IOs, 1841069056 bytes 2026-02-22T21:52:07.243 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4200 IOs, 1886682112 bytes 2026-02-22T21:52:07.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4300 IOs, 1902742528 bytes 2026-02-22T21:52:07.314 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4400 IOs, 1917204480 bytes 2026-02-22T21:52:07.374 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4500 IOs, 1979598848 bytes 2026-02-22T21:52:07.425 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4600 IOs, 2023518208 bytes 2026-02-22T21:52:07.469 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4700 IOs, 2051934208 bytes 2026-02-22T21:52:07.512 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4800 IOs, 2082339840 bytes 2026-02-22T21:52:07.554 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4900 IOs, 2115269632 bytes 2026-02-22T21:52:07.612 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5000 IOs, 2178307072 bytes 2026-02-22T21:52:07.670 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5100 IOs, 2251542528 bytes 2026-02-22T21:52:07.714 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5200 IOs, 2280105984 bytes 2026-02-22T21:52:07.779 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5300 IOs, 2361213952 bytes 2026-02-22T21:52:07.826 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5400 IOs, 2404770816 bytes 2026-02-22T21:52:07.880 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5500 IOs, 2457854976 bytes 2026-02-22T21:52:07.922 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5600 IOs, 2498022400 bytes 2026-02-22T21:52:07.954 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5700 IOs, 2509720576 bytes 2026-02-22T21:52:08.002 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5800 IOs, 2530593792 bytes 2026-02-22T21:52:08.056 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5900 IOs, 2583533568 bytes 2026-02-22T21:52:08.125 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6000 IOs, 2656513024 bytes 2026-02-22T21:52:08.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6100 IOs, 2753639424 bytes 2026-02-22T21:52:08.270 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6200 IOs, 2843282432 bytes 2026-02-22T21:52:08.311 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6300 IOs, 2865043456 bytes 2026-02-22T21:52:08.353 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6400 IOs, 2899213312 bytes 2026-02-22T21:52:08.401 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6500 IOs, 2936847360 bytes 2026-02-22T21:52:08.444 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6600 IOs, 2969919488 bytes 2026-02-22T21:52:08.477 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6700 IOs, 2976035840 bytes 2026-02-22T21:52:08.528 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6800 IOs, 3048893440 bytes 2026-02-22T21:52:08.582 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6900 IOs, 3070146560 bytes 2026-02-22T21:52:08.627 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7000 IOs, 3109274624 bytes 2026-02-22T21:52:08.681 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7100 IOs, 3162104832 bytes 2026-02-22T21:52:08.708 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7200 IOs, 3163542528 bytes 2026-02-22T21:52:08.749 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7300 IOs, 3191367680 bytes 2026-02-22T21:52:08.812 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7400 IOs, 3264609280 bytes 2026-02-22T21:52:08.877 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7500 IOs, 3290664960 bytes 2026-02-22T21:52:08.916 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7600 IOs, 3311050752 bytes 2026-02-22T21:52:08.951 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7700 IOs, 3320076288 bytes 2026-02-22T21:52:09.017 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7800 IOs, 3380142080 bytes 2026-02-22T21:52:09.070 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7900 IOs, 3428523008 bytes 2026-02-22T21:52:09.108 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8000 IOs, 3448418304 bytes 2026-02-22T21:52:09.158 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8100 IOs, 3498931200 bytes 2026-02-22T21:52:09.196 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8200 IOs, 3521563648 bytes 2026-02-22T21:52:09.444 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8300 IOs, 3559472128 bytes 2026-02-22T21:52:09.484 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8400 IOs, 3579078656 bytes 2026-02-22T21:52:09.528 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8500 IOs, 3611582464 bytes 2026-02-22T21:52:09.568 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8600 IOs, 3645627392 bytes 2026-02-22T21:52:09.627 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8700 IOs, 3712823296 bytes 2026-02-22T21:52:09.655 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8800 IOs, 3762028544 bytes 2026-02-22T21:52:09.719 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8900 IOs, 3805590528 bytes 2026-02-22T21:52:09.775 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9000 IOs, 3867114496 bytes 2026-02-22T21:52:09.836 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9100 IOs, 3939414016 bytes 2026-02-22T21:52:09.891 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9200 IOs, 3985549312 bytes 2026-02-22T21:52:09.954 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9300 IOs, 4044108800 bytes 2026-02-22T21:52:10.002 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9400 IOs, 4095248384 bytes 2026-02-22T21:52:10.075 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9500 IOs, 4152593408 bytes 2026-02-22T21:52:10.113 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9600 IOs, 4169046016 bytes 2026-02-22T21:52:10.161 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9700 IOs, 4197156864 bytes 2026-02-22T21:52:10.222 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9800 IOs, 4260839424 bytes 2026-02-22T21:52:10.267 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9900 IOs, 4311342080 bytes 2026-02-22T21:52:10.323 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10000 IOs, 4345247744 bytes 2026-02-22T21:52:10.359 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10100 IOs, 4359334912 bytes 2026-02-22T21:52:10.401 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10200 IOs, 4397146112 bytes 2026-02-22T21:52:10.460 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10300 IOs, 4455051264 bytes 2026-02-22T21:52:10.508 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10400 IOs, 4498866176 bytes 2026-02-22T21:52:10.559 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10500 IOs, 4530694144 bytes 2026-02-22T21:52:10.597 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10600 IOs, 4543719424 bytes 2026-02-22T21:52:10.642 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10700 IOs, 4568530944 bytes 2026-02-22T21:52:10.704 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10800 IOs, 4627162112 bytes 2026-02-22T21:52:10.747 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10900 IOs, 4666489856 bytes 2026-02-22T21:52:10.792 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11000 IOs, 4684163072 bytes 2026-02-22T21:52:10.823 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11100 IOs, 4684788736 bytes 2026-02-22T21:52:10.880 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11200 IOs, 4743490560 bytes 2026-02-22T21:52:10.928 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11300 IOs, 4782385152 bytes 2026-02-22T21:52:10.981 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11400 IOs, 4835320832 bytes 2026-02-22T21:52:11.042 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11500 IOs, 4888583168 bytes 2026-02-22T21:52:11.091 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11600 IOs, 4932555776 bytes 2026-02-22T21:52:11.145 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11700 IOs, 4975340544 bytes 2026-02-22T21:52:11.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11800 IOs, 5002184704 bytes 2026-02-22T21:52:11.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11900 IOs, 5036577792 bytes 2026-02-22T21:52:11.279 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12000 IOs, 5065549824 bytes 2026-02-22T21:52:11.330 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12100 IOs, 5125167104 bytes 2026-02-22T21:52:11.391 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12200 IOs, 5165359104 bytes 2026-02-22T21:52:11.465 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12300 IOs, 5248679936 bytes 2026-02-22T21:52:11.537 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12400 IOs, 5325618176 bytes 2026-02-22T21:52:11.583 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12500 IOs, 5356758016 bytes 2026-02-22T21:52:11.664 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12600 IOs, 5463703552 bytes 2026-02-22T21:52:11.703 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12700 IOs, 5490741248 bytes 2026-02-22T21:52:11.757 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12800 IOs, 5528574976 bytes 2026-02-22T21:52:11.807 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12900 IOs, 5577577472 bytes 2026-02-22T21:52:11.836 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13000 IOs, 5603376128 bytes 2026-02-22T21:52:11.872 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13100 IOs, 5603983360 bytes 2026-02-22T21:52:11.915 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13200 IOs, 5639960576 bytes 2026-02-22T21:52:11.986 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13300 IOs, 5702537216 bytes 2026-02-22T21:52:12.018 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13400 IOs, 5703159808 bytes 2026-02-22T21:52:12.079 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13500 IOs, 5741048832 bytes 2026-02-22T21:52:12.117 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13600 IOs, 5760549888 bytes 2026-02-22T21:52:12.171 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13700 IOs, 5781804032 bytes 2026-02-22T21:52:12.249 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13800 IOs, 5870609408 bytes 2026-02-22T21:52:12.319 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13900 IOs, 5942897664 bytes 2026-02-22T21:52:12.365 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14000 IOs, 5987335168 bytes 2026-02-22T21:52:12.407 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14100 IOs, 5993610240 bytes 2026-02-22T21:52:12.463 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14200 IOs, 6035491840 bytes 2026-02-22T21:52:12.502 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14300 IOs, 6043301888 bytes 2026-02-22T21:52:12.539 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14400 IOs, 6052201472 bytes 2026-02-22T21:52:12.594 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14500 IOs, 6101179392 bytes 2026-02-22T21:52:12.636 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14600 IOs, 6101786624 bytes 2026-02-22T21:52:12.683 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14700 IOs, 6127398912 bytes 2026-02-22T21:52:12.744 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14800 IOs, 6199206912 bytes 2026-02-22T21:52:12.791 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14900 IOs, 6224312320 bytes 2026-02-22T21:52:12.845 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15000 IOs, 6261526528 bytes 2026-02-22T21:52:12.945 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15100 IOs, 6389673984 bytes 2026-02-22T21:52:13.006 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15200 IOs, 6446900224 bytes 2026-02-22T21:52:13.053 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15300 IOs, 6474563584 bytes 2026-02-22T21:52:13.132 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15400 IOs, 6570489856 bytes 2026-02-22T21:52:13.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15500 IOs, 6625873920 bytes 2026-02-22T21:52:13.277 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15600 IOs, 6704125952 bytes 2026-02-22T21:52:13.342 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15700 IOs, 6768302080 bytes 2026-02-22T21:52:13.416 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15800 IOs, 6852494336 bytes 2026-02-22T21:52:13.459 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15900 IOs, 6871965696 bytes 2026-02-22T21:52:13.507 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16000 IOs, 6907549696 bytes 2026-02-22T21:52:13.545 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16100 IOs, 6926284800 bytes 2026-02-22T21:52:13.595 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16200 IOs, 6967134208 bytes 2026-02-22T21:52:13.643 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16300 IOs, 6997236736 bytes 2026-02-22T21:52:13.703 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16400 IOs, 7055876096 bytes 2026-02-22T21:52:13.778 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16500 IOs, 7139508224 bytes 2026-02-22T21:52:13.877 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16600 IOs, 7263669248 bytes 2026-02-22T21:52:13.910 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16700 IOs, 7264293888 bytes 2026-02-22T21:52:13.990 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16800 IOs, 7356464128 bytes 2026-02-22T21:52:14.042 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16900 IOs, 7392598016 bytes 2026-02-22T21:52:14.098 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17000 IOs, 7411954688 bytes 2026-02-22T21:52:14.153 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17100 IOs, 7451315200 bytes 2026-02-22T21:52:14.228 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17200 IOs, 7544327168 bytes 2026-02-22T21:52:14.292 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17300 IOs, 7606189056 bytes 2026-02-22T21:52:14.342 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17400 IOs, 7633312768 bytes 2026-02-22T21:52:14.402 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17500 IOs, 7686722560 bytes 2026-02-22T21:52:14.449 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17600 IOs, 7719463936 bytes 2026-02-22T21:52:14.495 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17700 IOs, 7743961088 bytes 2026-02-22T21:52:14.549 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17800 IOs, 7776388096 bytes 2026-02-22T21:52:14.602 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17900 IOs, 7809085440 bytes 2026-02-22T21:52:14.664 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18000 IOs, 7867586560 bytes 2026-02-22T21:52:14.709 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18100 IOs, 7890675712 bytes 2026-02-22T21:52:14.750 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18200 IOs, 7902580736 bytes 2026-02-22T21:52:14.783 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18300 IOs, 7903201280 bytes 2026-02-22T21:52:14.836 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18400 IOs, 7949024256 bytes 2026-02-22T21:52:14.883 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18500 IOs, 7974764544 bytes 2026-02-22T21:52:14.946 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18600 IOs, 8037253120 bytes 2026-02-22T21:52:15.005 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18700 IOs, 8111069184 bytes 2026-02-22T21:52:15.046 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18800 IOs, 8111682560 bytes 2026-02-22T21:52:15.111 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18900 IOs, 8157919232 bytes 2026-02-22T21:52:15.157 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19000 IOs, 8178878464 bytes 2026-02-22T21:52:15.203 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19100 IOs, 8204741632 bytes 2026-02-22T21:52:15.256 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19200 IOs, 8248017920 bytes 2026-02-22T21:52:15.311 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19300 IOs, 8291869696 bytes 2026-02-22T21:52:15.351 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19400 IOs, 8310255616 bytes 2026-02-22T21:52:15.405 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19500 IOs, 8346836992 bytes 2026-02-22T21:52:15.456 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19600 IOs, 8379836416 bytes 2026-02-22T21:52:15.524 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19700 IOs, 8453879808 bytes 2026-02-22T21:52:15.575 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19800 IOs, 8494546944 bytes 2026-02-22T21:52:15.630 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19900 IOs, 8533833728 bytes 2026-02-22T21:52:15.678 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20000 IOs, 8559050752 bytes 2026-02-22T21:52:15.740 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20100 IOs, 8619197440 bytes 2026-02-22T21:52:15.776 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20200 IOs, 8622814208 bytes 2026-02-22T21:52:15.812 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20300 IOs, 8642410496 bytes 2026-02-22T21:52:15.868 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20400 IOs, 8673821696 bytes 2026-02-22T21:52:15.918 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20500 IOs, 8698375168 bytes 2026-02-22T21:52:15.989 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20600 IOs, 8781327360 bytes 2026-02-22T21:52:16.024 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20700 IOs, 8786736128 bytes 2026-02-22T21:52:16.082 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20800 IOs, 8849132544 bytes 2026-02-22T21:52:16.141 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20900 IOs, 8868790272 bytes 2026-02-22T21:52:16.152 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ error_code=124 2026-02-22T21:52:16.153 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set -e 2026-02-22T21:52:16.153 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' 124 -eq 124 ']' 2026-02-22T21:52:16.153 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:52:16.153 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T21:52:16.153 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_26 2026-02-22T21:52:16.153 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image_and_enable_mirror cluster2 mirror image_26 snapshot 128M 2026-02-22T21:52:16.153 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:52:16.153 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:52:16.153 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:52:16.153 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:52:16.153 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_26 2026-02-22T21:52:16.153 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:52:16.153 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T21:52:16.153 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n snapshot ']' 2026-02-22T21:52:16.153 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:52:16.153 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image cluster2 mirror image_26 128M 2026-02-22T21:52:16.153 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:52:16.153 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:52:16.153 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:52:16.153 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:52:16.153 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_26 2026-02-22T21:52:16.153 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:52:16.153 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local size=128 2026-02-22T21:52:16.153 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n 128M ']' 2026-02-22T21:52:16.153 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ size=128M 2026-02-22T21:52:16.153 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:52:16.153 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_26 2026-02-22T21:52:16.184 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T21:52:16.184 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:52:16.184 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' image = image ']' 2026-02-22T21:52:16.184 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:52:16.184 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ enable_mirror cluster2 mirror image_26 snapshot 2026-02-22T21:52:16.184 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:52:16.184 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:52:16.184 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_26 2026-02-22T21:52:16.184 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T21:52:16.185 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_26 snapshot 2026-02-22T21:52:17.125 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Mirroring enabled 2026-02-22T21:52:17.130 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 info mirror/image_26 2026-02-22T21:52:17.155 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_26': 2026-02-22T21:52:17.155 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 128 MiB in 32 objects 2026-02-22T21:52:17.155 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T21:52:17.155 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 1 2026-02-22T21:52:17.155 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 138ae11c7ca9 2026-02-22T21:52:17.155 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.138ae11c7ca9 2026-02-22T21:52:17.155 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T21:52:17.155 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock 2026-02-22T21:52:17.155 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T21:52:17.155 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T21:52:17.155 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:52:16 2026 2026-02-22T21:52:17.155 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:52:16 2026 2026-02-22T21:52:17.155 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:52:16 2026 2026-02-22T21:52:17.155 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T21:52:17.155 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T21:52:17.155 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: f2b9fbe9-fa15-4b10-aead-5e1775df224b 2026-02-22T21:52:17.155 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: true 2026-02-22T21:52:17.157 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n '' ']' 2026-02-22T21:52:17.157 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ stress_write_image cluster2 mirror image_26 2026-02-22T21:52:17.157 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:52:17.157 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:52:17.157 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_26 2026-02-22T21:52:17.157 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-22T21:52:17.159 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local duration=5 2026-02-22T21:52:17.159 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set +e 2026-02-22T21:52:17.159 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ timeout 5s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_26 --debug-rbd=20 --debug-journaler=20 2026-02-22T21:52:17.243 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:100 IOs, 48908288 bytes 2026-02-22T21:52:17.320 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:200 IOs, 180142080 bytes 2026-02-22T21:52:17.390 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:300 IOs, 248545280 bytes 2026-02-22T21:52:17.426 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:400 IOs, 287889408 bytes 2026-02-22T21:52:17.466 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:500 IOs, 297165824 bytes 2026-02-22T21:52:17.508 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:600 IOs, 320921600 bytes 2026-02-22T21:52:17.572 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:700 IOs, 405225472 bytes 2026-02-22T21:52:17.627 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:800 IOs, 460749824 bytes 2026-02-22T21:52:17.662 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:900 IOs, 470691840 bytes 2026-02-22T21:52:17.707 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1000 IOs, 505043968 bytes 2026-02-22T21:52:17.757 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1100 IOs, 548424704 bytes 2026-02-22T21:52:17.802 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1200 IOs, 597479424 bytes 2026-02-22T21:52:17.862 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1300 IOs, 654514176 bytes 2026-02-22T21:52:17.914 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1400 IOs, 697059328 bytes 2026-02-22T21:52:17.957 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1500 IOs, 721939456 bytes 2026-02-22T21:52:18.022 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1600 IOs, 791585792 bytes 2026-02-22T21:52:18.056 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1700 IOs, 805531648 bytes 2026-02-22T21:52:18.108 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1800 IOs, 865340416 bytes 2026-02-22T21:52:18.153 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1900 IOs, 881700864 bytes 2026-02-22T21:52:18.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2000 IOs, 954167296 bytes 2026-02-22T21:52:18.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2100 IOs, 1001815040 bytes 2026-02-22T21:52:18.307 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2200 IOs, 1010107392 bytes 2026-02-22T21:52:18.364 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2300 IOs, 1054949376 bytes 2026-02-22T21:52:18.440 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2400 IOs, 1149563904 bytes 2026-02-22T21:52:18.491 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2500 IOs, 1184694272 bytes 2026-02-22T21:52:18.545 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2600 IOs, 1231754240 bytes 2026-02-22T21:52:18.614 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2700 IOs, 1320582144 bytes 2026-02-22T21:52:18.708 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2800 IOs, 1429338112 bytes 2026-02-22T21:52:18.754 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2900 IOs, 1455874048 bytes 2026-02-22T21:52:18.800 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3000 IOs, 1502029824 bytes 2026-02-22T21:52:18.861 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3100 IOs, 1557472256 bytes 2026-02-22T21:52:18.924 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3200 IOs, 1619338240 bytes 2026-02-22T21:52:18.984 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3300 IOs, 1687418880 bytes 2026-02-22T21:52:19.029 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3400 IOs, 1713440768 bytes 2026-02-22T21:52:19.094 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3500 IOs, 1789205504 bytes 2026-02-22T21:52:19.151 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3600 IOs, 1834866688 bytes 2026-02-22T21:52:19.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3700 IOs, 1853383680 bytes 2026-02-22T21:52:19.230 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3800 IOs, 1882060800 bytes 2026-02-22T21:52:19.262 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3900 IOs, 1886398464 bytes 2026-02-22T21:52:19.291 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4000 IOs, 1887097856 bytes 2026-02-22T21:52:19.357 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4100 IOs, 1945361408 bytes 2026-02-22T21:52:19.407 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4200 IOs, 1956322304 bytes 2026-02-22T21:52:19.459 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4300 IOs, 1997065216 bytes 2026-02-22T21:52:19.525 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4400 IOs, 2081650688 bytes 2026-02-22T21:52:19.559 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4500 IOs, 2091053056 bytes 2026-02-22T21:52:19.599 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4600 IOs, 2110352384 bytes 2026-02-22T21:52:19.660 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4700 IOs, 2183429120 bytes 2026-02-22T21:52:19.712 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4800 IOs, 2219313152 bytes 2026-02-22T21:52:19.746 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4900 IOs, 2229227520 bytes 2026-02-22T21:52:19.787 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5000 IOs, 2252800000 bytes 2026-02-22T21:52:19.840 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5100 IOs, 2294598656 bytes 2026-02-22T21:52:19.890 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5200 IOs, 2336915456 bytes 2026-02-22T21:52:19.938 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5300 IOs, 2372864000 bytes 2026-02-22T21:52:19.975 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5400 IOs, 2389903360 bytes 2026-02-22T21:52:20.016 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5500 IOs, 2414659584 bytes 2026-02-22T21:52:20.058 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5600 IOs, 2442739712 bytes 2026-02-22T21:52:20.090 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5700 IOs, 2447576064 bytes 2026-02-22T21:52:20.142 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5800 IOs, 2516492288 bytes 2026-02-22T21:52:20.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5900 IOs, 2598215680 bytes 2026-02-22T21:52:20.280 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6000 IOs, 2648530944 bytes 2026-02-22T21:52:20.320 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6100 IOs, 2690534400 bytes 2026-02-22T21:52:20.395 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6200 IOs, 2759751680 bytes 2026-02-22T21:52:20.454 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6300 IOs, 2825034752 bytes 2026-02-22T21:52:20.488 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6400 IOs, 2836230144 bytes 2026-02-22T21:52:20.527 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6500 IOs, 2877666304 bytes 2026-02-22T21:52:20.572 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6600 IOs, 2890951680 bytes 2026-02-22T21:52:20.600 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6700 IOs, 2891556864 bytes 2026-02-22T21:52:20.630 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6800 IOs, 2896106496 bytes 2026-02-22T21:52:20.687 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6900 IOs, 2957010944 bytes 2026-02-22T21:52:20.752 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7000 IOs, 3038333952 bytes 2026-02-22T21:52:20.788 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7100 IOs, 3057007616 bytes 2026-02-22T21:52:20.845 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7200 IOs, 3113285632 bytes 2026-02-22T21:52:20.908 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7300 IOs, 3190581248 bytes 2026-02-22T21:52:20.962 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7400 IOs, 3240769536 bytes 2026-02-22T21:52:21.021 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7500 IOs, 3315023872 bytes 2026-02-22T21:52:21.070 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7600 IOs, 3360796672 bytes 2026-02-22T21:52:21.137 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7700 IOs, 3440343040 bytes 2026-02-22T21:52:21.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7800 IOs, 3476157440 bytes 2026-02-22T21:52:21.240 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7900 IOs, 3512556544 bytes 2026-02-22T21:52:21.293 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8000 IOs, 3565864960 bytes 2026-02-22T21:52:21.355 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8100 IOs, 3646455808 bytes 2026-02-22T21:52:21.393 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8200 IOs, 3654703104 bytes 2026-02-22T21:52:21.441 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8300 IOs, 3691539456 bytes 2026-02-22T21:52:21.493 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8400 IOs, 3748672512 bytes 2026-02-22T21:52:21.563 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8500 IOs, 3817576448 bytes 2026-02-22T21:52:21.613 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8600 IOs, 3863625728 bytes 2026-02-22T21:52:21.649 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8700 IOs, 3878480896 bytes 2026-02-22T21:52:21.710 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8800 IOs, 3946387456 bytes 2026-02-22T21:52:21.746 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8900 IOs, 3956101120 bytes 2026-02-22T21:52:21.792 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9000 IOs, 3989747712 bytes 2026-02-22T21:52:21.835 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9100 IOs, 4014781440 bytes 2026-02-22T21:52:21.884 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9200 IOs, 4060685312 bytes 2026-02-22T21:52:21.930 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9300 IOs, 4102795264 bytes 2026-02-22T21:52:21.971 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9400 IOs, 4127424512 bytes 2026-02-22T21:52:22.011 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9500 IOs, 4143409152 bytes 2026-02-22T21:52:22.065 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9600 IOs, 4191005696 bytes 2026-02-22T21:52:22.106 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9700 IOs, 4226710528 bytes 2026-02-22T21:52:22.146 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9800 IOs, 4241446912 bytes 2026-02-22T21:52:22.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ error_code=124 2026-02-22T21:52:22.166 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set -e 2026-02-22T21:52:22.166 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' 124 -eq 124 ']' 2026-02-22T21:52:22.166 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:52:22.166 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T21:52:22.166 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_27 2026-02-22T21:52:22.166 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image_and_enable_mirror cluster2 mirror image_27 snapshot 128M 2026-02-22T21:52:22.166 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:52:22.166 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:52:22.166 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:52:22.166 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:52:22.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_27 2026-02-22T21:52:22.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:52:22.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T21:52:22.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n snapshot ']' 2026-02-22T21:52:22.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:52:22.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image cluster2 mirror image_27 128M 2026-02-22T21:52:22.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:52:22.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:52:22.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:52:22.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:52:22.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_27 2026-02-22T21:52:22.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:52:22.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local size=128 2026-02-22T21:52:22.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n 128M ']' 2026-02-22T21:52:22.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ size=128M 2026-02-22T21:52:22.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:52:22.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_27 2026-02-22T21:52:22.195 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T21:52:22.195 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:52:22.195 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' image = image ']' 2026-02-22T21:52:22.195 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:52:22.195 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ enable_mirror cluster2 mirror image_27 snapshot 2026-02-22T21:52:22.195 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:52:22.195 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:52:22.196 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_27 2026-02-22T21:52:22.196 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T21:52:22.196 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_27 snapshot 2026-02-22T21:52:23.125 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Mirroring enabled 2026-02-22T21:52:23.130 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 info mirror/image_27 2026-02-22T21:52:23.154 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_27': 2026-02-22T21:52:23.154 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 128 MiB in 32 objects 2026-02-22T21:52:23.155 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T21:52:23.155 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 1 2026-02-22T21:52:23.155 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 139427a36c37 2026-02-22T21:52:23.155 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.139427a36c37 2026-02-22T21:52:23.155 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T21:52:23.155 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock 2026-02-22T21:52:23.155 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T21:52:23.155 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T21:52:23.155 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:52:22 2026 2026-02-22T21:52:23.155 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:52:22 2026 2026-02-22T21:52:23.155 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:52:22 2026 2026-02-22T21:52:23.155 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T21:52:23.156 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T21:52:23.156 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: 226f7166-427f-4ef9-893a-6179cde62b6d 2026-02-22T21:52:23.156 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: true 2026-02-22T21:52:23.157 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n '' ']' 2026-02-22T21:52:23.157 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ stress_write_image cluster2 mirror image_27 2026-02-22T21:52:23.157 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:52:23.157 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:52:23.157 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_27 2026-02-22T21:52:23.157 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-22T21:52:23.158 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local duration=12 2026-02-22T21:52:23.158 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set +e 2026-02-22T21:52:23.158 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ timeout 12s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_27 --debug-rbd=20 --debug-journaler=20 2026-02-22T21:52:23.242 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:100 IOs, 52286464 bytes 2026-02-22T21:52:23.287 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:200 IOs, 90699776 bytes 2026-02-22T21:52:23.347 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:300 IOs, 169434112 bytes 2026-02-22T21:52:23.385 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:400 IOs, 189372416 bytes 2026-02-22T21:52:23.444 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:500 IOs, 250981376 bytes 2026-02-22T21:52:23.488 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:600 IOs, 286183424 bytes 2026-02-22T21:52:23.541 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:700 IOs, 341491712 bytes 2026-02-22T21:52:23.624 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:800 IOs, 476874752 bytes 2026-02-22T21:52:23.696 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:900 IOs, 558678016 bytes 2026-02-22T21:52:23.730 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1000 IOs, 575836160 bytes 2026-02-22T21:52:23.793 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1100 IOs, 653935616 bytes 2026-02-22T21:52:23.838 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1200 IOs, 705166336 bytes 2026-02-22T21:52:23.894 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1300 IOs, 771688448 bytes 2026-02-22T21:52:23.950 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1400 IOs, 811817984 bytes 2026-02-22T21:52:23.994 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1500 IOs, 842157056 bytes 2026-02-22T21:52:24.041 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1600 IOs, 892175360 bytes 2026-02-22T21:52:24.097 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1700 IOs, 938551296 bytes 2026-02-22T21:52:24.162 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1800 IOs, 1006729216 bytes 2026-02-22T21:52:24.236 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1900 IOs, 1109470208 bytes 2026-02-22T21:52:24.283 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2000 IOs, 1144346624 bytes 2026-02-22T21:52:24.326 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2100 IOs, 1158590464 bytes 2026-02-22T21:52:24.372 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2200 IOs, 1196345344 bytes 2026-02-22T21:52:24.420 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2300 IOs, 1234170880 bytes 2026-02-22T21:52:24.487 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2400 IOs, 1308392448 bytes 2026-02-22T21:52:24.514 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2500 IOs, 1308994560 bytes 2026-02-22T21:52:24.555 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2600 IOs, 1340283904 bytes 2026-02-22T21:52:24.612 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2700 IOs, 1396525056 bytes 2026-02-22T21:52:24.665 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2800 IOs, 1463752704 bytes 2026-02-22T21:52:24.726 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2900 IOs, 1528197120 bytes 2026-02-22T21:52:24.775 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3000 IOs, 1572161536 bytes 2026-02-22T21:52:24.821 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3100 IOs, 1611718656 bytes 2026-02-22T21:52:24.865 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3200 IOs, 1642430464 bytes 2026-02-22T21:52:24.899 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3300 IOs, 1672356864 bytes 2026-02-22T21:52:24.952 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3400 IOs, 1704084480 bytes 2026-02-22T21:52:24.981 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3500 IOs, 1704706048 bytes 2026-02-22T21:52:25.009 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3600 IOs, 1708413952 bytes 2026-02-22T21:52:25.042 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3700 IOs, 1717858304 bytes 2026-02-22T21:52:25.094 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3800 IOs, 1768545280 bytes 2026-02-22T21:52:25.134 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3900 IOs, 1787722752 bytes 2026-02-22T21:52:25.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4000 IOs, 1847683072 bytes 2026-02-22T21:52:25.262 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4100 IOs, 1911661568 bytes 2026-02-22T21:52:25.289 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4200 IOs, 1913499648 bytes 2026-02-22T21:52:25.343 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4300 IOs, 1966811136 bytes 2026-02-22T21:52:25.393 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4400 IOs, 1997435904 bytes 2026-02-22T21:52:25.441 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4500 IOs, 2039021568 bytes 2026-02-22T21:52:25.473 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4600 IOs, 2047616000 bytes 2026-02-22T21:52:25.549 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4700 IOs, 2157853696 bytes 2026-02-22T21:52:25.589 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4800 IOs, 2177881088 bytes 2026-02-22T21:52:25.648 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4900 IOs, 2246181888 bytes 2026-02-22T21:52:25.745 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5000 IOs, 2399499264 bytes 2026-02-22T21:52:25.782 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5100 IOs, 2426465280 bytes 2026-02-22T21:52:25.846 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5200 IOs, 2500961280 bytes 2026-02-22T21:52:25.899 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5300 IOs, 2552666112 bytes 2026-02-22T21:52:25.929 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5400 IOs, 2557118464 bytes 2026-02-22T21:52:25.987 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5500 IOs, 2610139136 bytes 2026-02-22T21:52:26.026 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5600 IOs, 2627420160 bytes 2026-02-22T21:52:26.092 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5700 IOs, 2718315520 bytes 2026-02-22T21:52:26.147 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5800 IOs, 2781739008 bytes 2026-02-22T21:52:26.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5900 IOs, 2788215808 bytes 2026-02-22T21:52:26.226 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6000 IOs, 2826165248 bytes 2026-02-22T21:52:26.327 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6100 IOs, 2977913856 bytes 2026-02-22T21:52:26.373 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6200 IOs, 2997713920 bytes 2026-02-22T21:52:26.434 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6300 IOs, 3053102080 bytes 2026-02-22T21:52:26.480 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6400 IOs, 3093595136 bytes 2026-02-22T21:52:26.518 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6500 IOs, 3119776768 bytes 2026-02-22T21:52:26.566 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6600 IOs, 3158522880 bytes 2026-02-22T21:52:26.604 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6700 IOs, 3178075136 bytes 2026-02-22T21:52:26.660 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6800 IOs, 3238948864 bytes 2026-02-22T21:52:26.708 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6900 IOs, 3283753984 bytes 2026-02-22T21:52:26.761 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7000 IOs, 3333606400 bytes 2026-02-22T21:52:26.793 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7100 IOs, 3345050624 bytes 2026-02-22T21:52:26.819 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7200 IOs, 3345696768 bytes 2026-02-22T21:52:26.887 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7300 IOs, 3425809408 bytes 2026-02-22T21:52:26.934 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7400 IOs, 3467588608 bytes 2026-02-22T21:52:26.965 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7500 IOs, 3468204032 bytes 2026-02-22T21:52:27.018 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7600 IOs, 3512728576 bytes 2026-02-22T21:52:27.051 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7700 IOs, 3525356544 bytes 2026-02-22T21:52:27.083 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7800 IOs, 3548858368 bytes 2026-02-22T21:52:27.127 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7900 IOs, 3581765632 bytes 2026-02-22T21:52:27.170 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8000 IOs, 3603687424 bytes 2026-02-22T21:52:27.226 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8100 IOs, 3664402432 bytes 2026-02-22T21:52:27.302 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8200 IOs, 3747786752 bytes 2026-02-22T21:52:27.339 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8300 IOs, 3762100224 bytes 2026-02-22T21:52:27.386 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8400 IOs, 3805612032 bytes 2026-02-22T21:52:27.413 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8500 IOs, 3806243840 bytes 2026-02-22T21:52:27.459 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8600 IOs, 3866215424 bytes 2026-02-22T21:52:27.526 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8700 IOs, 3933060096 bytes 2026-02-22T21:52:27.576 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8800 IOs, 3971153920 bytes 2026-02-22T21:52:27.630 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8900 IOs, 4034432000 bytes 2026-02-22T21:52:27.669 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9000 IOs, 4042519552 bytes 2026-02-22T21:52:27.713 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9100 IOs, 4075814912 bytes 2026-02-22T21:52:27.775 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9200 IOs, 4145523712 bytes 2026-02-22T21:52:27.828 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9300 IOs, 4201711616 bytes 2026-02-22T21:52:27.862 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9400 IOs, 4232491008 bytes 2026-02-22T21:52:27.925 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9500 IOs, 4288549888 bytes 2026-02-22T21:52:27.984 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9600 IOs, 4359114752 bytes 2026-02-22T21:52:28.026 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9700 IOs, 4368927744 bytes 2026-02-22T21:52:28.074 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9800 IOs, 4415724544 bytes 2026-02-22T21:52:28.102 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9900 IOs, 4416355328 bytes 2026-02-22T21:52:28.159 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10000 IOs, 4467056640 bytes 2026-02-22T21:52:28.195 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10100 IOs, 4486083584 bytes 2026-02-22T21:52:28.264 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10200 IOs, 4561855488 bytes 2026-02-22T21:52:28.311 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10300 IOs, 4607907840 bytes 2026-02-22T21:52:28.365 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10400 IOs, 4663546880 bytes 2026-02-22T21:52:28.401 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10500 IOs, 4678407168 bytes 2026-02-22T21:52:28.452 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10600 IOs, 4724879360 bytes 2026-02-22T21:52:28.503 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10700 IOs, 4766972928 bytes 2026-02-22T21:52:28.577 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10800 IOs, 4853603328 bytes 2026-02-22T21:52:28.603 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10900 IOs, 4855216128 bytes 2026-02-22T21:52:28.638 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11000 IOs, 4877496320 bytes 2026-02-22T21:52:28.699 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11100 IOs, 4948936704 bytes 2026-02-22T21:52:28.744 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11200 IOs, 4973250560 bytes 2026-02-22T21:52:28.788 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11300 IOs, 5008859136 bytes 2026-02-22T21:52:28.833 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11400 IOs, 5035468800 bytes 2026-02-22T21:52:28.881 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11500 IOs, 5079013376 bytes 2026-02-22T21:52:28.929 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11600 IOs, 5128208384 bytes 2026-02-22T21:52:28.980 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11700 IOs, 5169415168 bytes 2026-02-22T21:52:29.022 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11800 IOs, 5189023744 bytes 2026-02-22T21:52:29.069 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11900 IOs, 5250756608 bytes 2026-02-22T21:52:29.128 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12000 IOs, 5292180480 bytes 2026-02-22T21:52:29.174 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12100 IOs, 5329387520 bytes 2026-02-22T21:52:29.225 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12200 IOs, 5368189952 bytes 2026-02-22T21:52:29.287 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12300 IOs, 5428499456 bytes 2026-02-22T21:52:29.361 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12400 IOs, 5518881792 bytes 2026-02-22T21:52:29.420 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12500 IOs, 5579592704 bytes 2026-02-22T21:52:29.491 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12600 IOs, 5650557952 bytes 2026-02-22T21:52:29.534 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12700 IOs, 5682066432 bytes 2026-02-22T21:52:29.564 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12800 IOs, 5693701120 bytes 2026-02-22T21:52:29.614 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12900 IOs, 5744458752 bytes 2026-02-22T21:52:29.662 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13000 IOs, 5783334912 bytes 2026-02-22T21:52:29.734 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13100 IOs, 5872406528 bytes 2026-02-22T21:52:29.796 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13200 IOs, 5946391552 bytes 2026-02-22T21:52:29.833 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13300 IOs, 5968034816 bytes 2026-02-22T21:52:29.877 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13400 IOs, 6006339584 bytes 2026-02-22T21:52:29.923 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13500 IOs, 6061186048 bytes 2026-02-22T21:52:29.966 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13600 IOs, 6082277376 bytes 2026-02-22T21:52:30.008 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13700 IOs, 6112952320 bytes 2026-02-22T21:52:30.080 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13800 IOs, 6202101760 bytes 2026-02-22T21:52:30.111 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13900 IOs, 6207012864 bytes 2026-02-22T21:52:30.163 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14000 IOs, 6253964288 bytes 2026-02-22T21:52:30.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14100 IOs, 6326724608 bytes 2026-02-22T21:52:30.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14200 IOs, 6361107456 bytes 2026-02-22T21:52:30.321 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14300 IOs, 6392360960 bytes 2026-02-22T21:52:30.356 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14400 IOs, 6423308288 bytes 2026-02-22T21:52:30.403 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14500 IOs, 6449666048 bytes 2026-02-22T21:52:30.452 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14600 IOs, 6485336064 bytes 2026-02-22T21:52:30.513 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14700 IOs, 6551283712 bytes 2026-02-22T21:52:30.555 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14800 IOs, 6574550016 bytes 2026-02-22T21:52:30.600 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14900 IOs, 6609854464 bytes 2026-02-22T21:52:30.627 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15000 IOs, 6610478080 bytes 2026-02-22T21:52:30.666 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15100 IOs, 6629201920 bytes 2026-02-22T21:52:30.731 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15200 IOs, 6719157248 bytes 2026-02-22T21:52:30.780 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15300 IOs, 6750731264 bytes 2026-02-22T21:52:30.830 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15400 IOs, 6792830976 bytes 2026-02-22T21:52:30.861 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15500 IOs, 6793428992 bytes 2026-02-22T21:52:30.912 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15600 IOs, 6839581696 bytes 2026-02-22T21:52:30.959 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15700 IOs, 6897203200 bytes 2026-02-22T21:52:30.996 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15800 IOs, 6897811456 bytes 2026-02-22T21:52:31.064 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15900 IOs, 6963127296 bytes 2026-02-22T21:52:31.125 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16000 IOs, 7029851136 bytes 2026-02-22T21:52:31.166 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16100 IOs, 7064049664 bytes 2026-02-22T21:52:31.219 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16200 IOs, 7113161728 bytes 2026-02-22T21:52:31.257 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16300 IOs, 7133452288 bytes 2026-02-22T21:52:31.308 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16400 IOs, 7180033024 bytes 2026-02-22T21:52:31.346 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16500 IOs, 7202293760 bytes 2026-02-22T21:52:31.402 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16600 IOs, 7262211072 bytes 2026-02-22T21:52:31.473 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16700 IOs, 7333704704 bytes 2026-02-22T21:52:31.500 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16800 IOs, 7353802752 bytes 2026-02-22T21:52:31.552 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16900 IOs, 7389617152 bytes 2026-02-22T21:52:31.601 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17000 IOs, 7433499648 bytes 2026-02-22T21:52:31.647 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17100 IOs, 7457733632 bytes 2026-02-22T21:52:31.697 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17200 IOs, 7497509888 bytes 2026-02-22T21:52:31.736 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17300 IOs, 7525526528 bytes 2026-02-22T21:52:31.804 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17400 IOs, 7622575104 bytes 2026-02-22T21:52:31.853 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17500 IOs, 7646095360 bytes 2026-02-22T21:52:31.895 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17600 IOs, 7672850432 bytes 2026-02-22T21:52:31.951 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17700 IOs, 7724870656 bytes 2026-02-22T21:52:32.013 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17800 IOs, 7810286592 bytes 2026-02-22T21:52:32.082 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17900 IOs, 7870197760 bytes 2026-02-22T21:52:32.125 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18000 IOs, 7894856704 bytes 2026-02-22T21:52:32.159 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18100 IOs, 7909638144 bytes 2026-02-22T21:52:32.215 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18200 IOs, 7959930880 bytes 2026-02-22T21:52:32.264 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18300 IOs, 8016707584 bytes 2026-02-22T21:52:32.338 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18400 IOs, 8086190080 bytes 2026-02-22T21:52:32.375 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18500 IOs, 8100837376 bytes 2026-02-22T21:52:32.404 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18600 IOs, 8108597248 bytes 2026-02-22T21:52:32.439 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18700 IOs, 8122876928 bytes 2026-02-22T21:52:32.492 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18800 IOs, 8168370176 bytes 2026-02-22T21:52:32.534 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18900 IOs, 8201511936 bytes 2026-02-22T21:52:32.581 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19000 IOs, 8251471872 bytes 2026-02-22T21:52:32.610 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19100 IOs, 8259806208 bytes 2026-02-22T21:52:32.635 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19200 IOs, 8260433920 bytes 2026-02-22T21:52:32.704 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19300 IOs, 8338964480 bytes 2026-02-22T21:52:32.753 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19400 IOs, 8379859968 bytes 2026-02-22T21:52:32.812 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19500 IOs, 8454604800 bytes 2026-02-22T21:52:32.862 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19600 IOs, 8479199232 bytes 2026-02-22T21:52:32.913 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19700 IOs, 8538335232 bytes 2026-02-22T21:52:32.988 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19800 IOs, 8621426688 bytes 2026-02-22T21:52:33.018 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19900 IOs, 8622066688 bytes 2026-02-22T21:52:33.085 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20000 IOs, 8689516544 bytes 2026-02-22T21:52:33.123 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20100 IOs, 8708313088 bytes 2026-02-22T21:52:33.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20200 IOs, 8771830784 bytes 2026-02-22T21:52:33.228 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20300 IOs, 8810105856 bytes 2026-02-22T21:52:33.274 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20400 IOs, 8864362496 bytes 2026-02-22T21:52:33.332 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20500 IOs, 8895121408 bytes 2026-02-22T21:52:33.364 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20600 IOs, 8900291584 bytes 2026-02-22T21:52:33.411 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20700 IOs, 8938658816 bytes 2026-02-22T21:52:33.451 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20800 IOs, 8968992768 bytes 2026-02-22T21:52:33.505 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20900 IOs, 9019719680 bytes 2026-02-22T21:52:33.562 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21000 IOs, 9066167296 bytes 2026-02-22T21:52:33.629 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21100 IOs, 9143825408 bytes 2026-02-22T21:52:33.656 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21200 IOs, 9144434688 bytes 2026-02-22T21:52:33.697 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21300 IOs, 9184417792 bytes 2026-02-22T21:52:33.745 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21400 IOs, 9224863744 bytes 2026-02-22T21:52:33.782 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21500 IOs, 9226150912 bytes 2026-02-22T21:52:33.856 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21600 IOs, 9319514112 bytes 2026-02-22T21:52:33.899 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21700 IOs, 9345675264 bytes 2026-02-22T21:52:33.942 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21800 IOs, 9374145536 bytes 2026-02-22T21:52:33.977 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21900 IOs, 9391332352 bytes 2026-02-22T21:52:34.029 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22000 IOs, 9433279488 bytes 2026-02-22T21:52:34.074 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22100 IOs, 9459816448 bytes 2026-02-22T21:52:34.141 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22200 IOs, 9540559872 bytes 2026-02-22T21:52:34.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22300 IOs, 9623085056 bytes 2026-02-22T21:52:34.262 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22400 IOs, 9668937728 bytes 2026-02-22T21:52:34.332 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22500 IOs, 9748410368 bytes 2026-02-22T21:52:34.362 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22600 IOs, 9754561536 bytes 2026-02-22T21:52:34.403 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22700 IOs, 9779493888 bytes 2026-02-22T21:52:34.464 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22800 IOs, 9843981312 bytes 2026-02-22T21:52:34.499 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22900 IOs, 9872145408 bytes 2026-02-22T21:52:34.567 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23000 IOs, 9965313024 bytes 2026-02-22T21:52:34.628 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23100 IOs, 10013157376 bytes 2026-02-22T21:52:34.686 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23200 IOs, 10098557952 bytes 2026-02-22T21:52:34.727 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23300 IOs, 10113858560 bytes 2026-02-22T21:52:34.768 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23400 IOs, 10147646464 bytes 2026-02-22T21:52:34.809 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23500 IOs, 10172229632 bytes 2026-02-22T21:52:34.850 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23600 IOs, 10200521728 bytes 2026-02-22T21:52:34.896 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23700 IOs, 10238744576 bytes 2026-02-22T21:52:34.937 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23800 IOs, 10262002688 bytes 2026-02-22T21:52:34.998 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23900 IOs, 10321473536 bytes 2026-02-22T21:52:35.065 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24000 IOs, 10402462720 bytes 2026-02-22T21:52:35.136 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24100 IOs, 10455605248 bytes 2026-02-22T21:52:35.163 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ error_code=124 2026-02-22T21:52:35.163 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set -e 2026-02-22T21:52:35.163 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' 124 -eq 124 ']' 2026-02-22T21:52:35.163 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:52:35.163 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T21:52:35.163 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_28 2026-02-22T21:52:35.163 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image_and_enable_mirror cluster2 mirror image_28 snapshot 128M 2026-02-22T21:52:35.163 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:52:35.163 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:52:35.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:52:35.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:52:35.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_28 2026-02-22T21:52:35.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:52:35.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T21:52:35.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n snapshot ']' 2026-02-22T21:52:35.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:52:35.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image cluster2 mirror image_28 128M 2026-02-22T21:52:35.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:52:35.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:52:35.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:52:35.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:52:35.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_28 2026-02-22T21:52:35.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:52:35.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local size=128 2026-02-22T21:52:35.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n 128M ']' 2026-02-22T21:52:35.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ size=128M 2026-02-22T21:52:35.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:52:35.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_28 2026-02-22T21:52:35.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T21:52:35.195 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:52:35.195 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' image = image ']' 2026-02-22T21:52:35.195 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:52:35.195 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ enable_mirror cluster2 mirror image_28 snapshot 2026-02-22T21:52:35.195 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:52:35.195 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:52:35.195 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_28 2026-02-22T21:52:35.195 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T21:52:35.195 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_28 snapshot 2026-02-22T21:52:36.136 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Mirroring enabled 2026-02-22T21:52:36.141 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 info mirror/image_28 2026-02-22T21:52:36.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_28': 2026-02-22T21:52:36.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 128 MiB in 32 objects 2026-02-22T21:52:36.166 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T21:52:36.166 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 1 2026-02-22T21:52:36.166 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 139e43087f89 2026-02-22T21:52:36.166 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.139e43087f89 2026-02-22T21:52:36.166 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T21:52:36.166 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock 2026-02-22T21:52:36.166 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T21:52:36.166 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T21:52:36.166 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:52:35 2026 2026-02-22T21:52:36.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:52:35 2026 2026-02-22T21:52:36.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:52:35 2026 2026-02-22T21:52:36.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T21:52:36.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T21:52:36.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: 8b05d225-eedc-4b0a-94f7-6d3c0ca41d35 2026-02-22T21:52:36.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: true 2026-02-22T21:52:36.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n '' ']' 2026-02-22T21:52:36.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ stress_write_image cluster2 mirror image_28 2026-02-22T21:52:36.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:52:36.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:52:36.169 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_28 2026-02-22T21:52:36.169 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-22T21:52:36.169 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local duration=6 2026-02-22T21:52:36.169 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set +e 2026-02-22T21:52:36.169 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ timeout 6s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_28 --debug-rbd=20 --debug-journaler=20 2026-02-22T21:52:36.234 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:100 IOs, 12430336 bytes 2026-02-22T21:52:36.293 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:200 IOs, 90343424 bytes 2026-02-22T21:52:36.348 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:300 IOs, 123717632 bytes 2026-02-22T21:52:36.381 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:400 IOs, 147781632 bytes 2026-02-22T21:52:36.437 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:500 IOs, 198845440 bytes 2026-02-22T21:52:36.477 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:600 IOs, 221928448 bytes 2026-02-22T21:52:36.738 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:700 IOs, 285679616 bytes 2026-02-22T21:52:36.783 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:800 IOs, 320129024 bytes 2026-02-22T21:52:36.820 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:900 IOs, 339288064 bytes 2026-02-22T21:52:36.862 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1000 IOs, 364587008 bytes 2026-02-22T21:52:36.895 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1100 IOs, 380524544 bytes 2026-02-22T21:52:36.932 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1200 IOs, 405801984 bytes 2026-02-22T21:52:36.976 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1300 IOs, 426949632 bytes 2026-02-22T21:52:37.011 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1400 IOs, 451314688 bytes 2026-02-22T21:52:37.058 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1500 IOs, 486955008 bytes 2026-02-22T21:52:37.105 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1600 IOs, 507246592 bytes 2026-02-22T21:52:37.153 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1700 IOs, 556116992 bytes 2026-02-22T21:52:37.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1800 IOs, 584753152 bytes 2026-02-22T21:52:37.231 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1900 IOs, 592049152 bytes 2026-02-22T21:52:37.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2000 IOs, 648331264 bytes 2026-02-22T21:52:37.324 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2100 IOs, 668831744 bytes 2026-02-22T21:52:37.363 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2200 IOs, 674279424 bytes 2026-02-22T21:52:37.420 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2300 IOs, 732413952 bytes 2026-02-22T21:52:37.476 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2400 IOs, 790630400 bytes 2026-02-22T21:52:37.510 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2500 IOs, 802890752 bytes 2026-02-22T21:52:37.542 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2600 IOs, 809859072 bytes 2026-02-22T21:52:37.602 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2700 IOs, 875817984 bytes 2026-02-22T21:52:37.643 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2800 IOs, 907792384 bytes 2026-02-22T21:52:37.718 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2900 IOs, 1002264576 bytes 2026-02-22T21:52:37.792 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3000 IOs, 1101121536 bytes 2026-02-22T21:52:37.845 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3100 IOs, 1164694528 bytes 2026-02-22T21:52:37.886 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3200 IOs, 1180540928 bytes 2026-02-22T21:52:37.919 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3300 IOs, 1205225472 bytes 2026-02-22T21:52:37.988 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3400 IOs, 1290470400 bytes 2026-02-22T21:52:38.064 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3500 IOs, 1376611328 bytes 2026-02-22T21:52:38.097 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3600 IOs, 1389749248 bytes 2026-02-22T21:52:38.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3700 IOs, 1490505728 bytes 2026-02-22T21:52:38.215 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3800 IOs, 1513180160 bytes 2026-02-22T21:52:38.255 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3900 IOs, 1547928576 bytes 2026-02-22T21:52:38.304 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4000 IOs, 1593310208 bytes 2026-02-22T21:52:38.357 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4100 IOs, 1626770432 bytes 2026-02-22T21:52:38.412 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4200 IOs, 1680158720 bytes 2026-02-22T21:52:38.474 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4300 IOs, 1748524032 bytes 2026-02-22T21:52:38.527 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4400 IOs, 1801029632 bytes 2026-02-22T21:52:38.554 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4500 IOs, 1803190272 bytes 2026-02-22T21:52:38.582 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4600 IOs, 1807384576 bytes 2026-02-22T21:52:38.626 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4700 IOs, 1841935360 bytes 2026-02-22T21:52:38.682 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4800 IOs, 1893226496 bytes 2026-02-22T21:52:38.715 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4900 IOs, 1904983040 bytes 2026-02-22T21:52:38.763 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5000 IOs, 1947524096 bytes 2026-02-22T21:52:38.804 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5100 IOs, 1982194688 bytes 2026-02-22T21:52:38.839 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5200 IOs, 1987796992 bytes 2026-02-22T21:52:38.875 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5300 IOs, 2006174720 bytes 2026-02-22T21:52:38.909 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5400 IOs, 2026884096 bytes 2026-02-22T21:52:38.955 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5500 IOs, 2054777856 bytes 2026-02-22T21:52:39.003 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5600 IOs, 2115095552 bytes 2026-02-22T21:52:39.045 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5700 IOs, 2119058432 bytes 2026-02-22T21:52:39.082 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5800 IOs, 2137949184 bytes 2026-02-22T21:52:39.108 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5900 IOs, 2139036672 bytes 2026-02-22T21:52:39.161 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6000 IOs, 2183303168 bytes 2026-02-22T21:52:39.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6100 IOs, 2204648448 bytes 2026-02-22T21:52:39.240 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6200 IOs, 2240963584 bytes 2026-02-22T21:52:39.274 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6300 IOs, 2258740224 bytes 2026-02-22T21:52:39.342 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6400 IOs, 2339177472 bytes 2026-02-22T21:52:39.392 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6500 IOs, 2365097984 bytes 2026-02-22T21:52:39.422 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6600 IOs, 2365705216 bytes 2026-02-22T21:52:39.460 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6700 IOs, 2390051840 bytes 2026-02-22T21:52:39.508 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6800 IOs, 2443821056 bytes 2026-02-22T21:52:39.573 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6900 IOs, 2505150464 bytes 2026-02-22T21:52:39.625 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7000 IOs, 2551745536 bytes 2026-02-22T21:52:39.676 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7100 IOs, 2605447168 bytes 2026-02-22T21:52:39.726 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7200 IOs, 2643682304 bytes 2026-02-22T21:52:39.808 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7300 IOs, 2764082176 bytes 2026-02-22T21:52:39.855 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7400 IOs, 2792841216 bytes 2026-02-22T21:52:39.910 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7500 IOs, 2861728768 bytes 2026-02-22T21:52:39.982 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7600 IOs, 2939438080 bytes 2026-02-22T21:52:40.014 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7700 IOs, 2964811776 bytes 2026-02-22T21:52:40.061 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7800 IOs, 2993352704 bytes 2026-02-22T21:52:40.108 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7900 IOs, 3042232320 bytes 2026-02-22T21:52:40.169 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8000 IOs, 3099553792 bytes 2026-02-22T21:52:40.224 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8100 IOs, 3157453824 bytes 2026-02-22T21:52:40.260 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8200 IOs, 3167535104 bytes 2026-02-22T21:52:40.294 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8300 IOs, 3179752448 bytes 2026-02-22T21:52:40.348 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8400 IOs, 3227529216 bytes 2026-02-22T21:52:40.397 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8500 IOs, 3255793664 bytes 2026-02-22T21:52:40.446 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8600 IOs, 3292788736 bytes 2026-02-22T21:52:40.513 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8700 IOs, 3377683456 bytes 2026-02-22T21:52:40.560 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8800 IOs, 3402547200 bytes 2026-02-22T21:52:40.596 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8900 IOs, 3417710592 bytes 2026-02-22T21:52:40.629 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9000 IOs, 3429108736 bytes 2026-02-22T21:52:40.668 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9100 IOs, 3462534144 bytes 2026-02-22T21:52:40.731 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9200 IOs, 3538613248 bytes 2026-02-22T21:52:40.808 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9300 IOs, 3651678208 bytes 2026-02-22T21:52:40.842 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9400 IOs, 3652308992 bytes 2026-02-22T21:52:40.904 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9500 IOs, 3732362240 bytes 2026-02-22T21:52:40.982 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9600 IOs, 3820512256 bytes 2026-02-22T21:52:41.025 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9700 IOs, 3850491904 bytes 2026-02-22T21:52:41.083 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9800 IOs, 3913133056 bytes 2026-02-22T21:52:41.132 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9900 IOs, 3951212544 bytes 2026-02-22T21:52:41.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10000 IOs, 3999292416 bytes 2026-02-22T21:52:41.231 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10100 IOs, 4045558784 bytes 2026-02-22T21:52:41.298 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10200 IOs, 4134321152 bytes 2026-02-22T21:52:41.326 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10300 IOs, 4134913024 bytes 2026-02-22T21:52:41.365 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10400 IOs, 4178196480 bytes 2026-02-22T21:52:41.459 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10500 IOs, 4274037760 bytes 2026-02-22T21:52:41.518 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10600 IOs, 4329543680 bytes 2026-02-22T21:52:41.560 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10700 IOs, 4352716800 bytes 2026-02-22T21:52:41.609 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10800 IOs, 4414579712 bytes 2026-02-22T21:52:41.663 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10900 IOs, 4447088640 bytes 2026-02-22T21:52:41.702 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11000 IOs, 4464079872 bytes 2026-02-22T21:52:41.748 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11100 IOs, 4504372224 bytes 2026-02-22T21:52:41.784 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11200 IOs, 4522155008 bytes 2026-02-22T21:52:41.825 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11300 IOs, 4546045952 bytes 2026-02-22T21:52:41.860 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11400 IOs, 4552740864 bytes 2026-02-22T21:52:41.890 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11500 IOs, 4573672448 bytes 2026-02-22T21:52:41.951 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11600 IOs, 4620806144 bytes 2026-02-22T21:52:42.013 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11700 IOs, 4704557056 bytes 2026-02-22T21:52:42.067 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11800 IOs, 4748853248 bytes 2026-02-22T21:52:42.113 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11900 IOs, 4780824576 bytes 2026-02-22T21:52:42.150 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12000 IOs, 4794850304 bytes 2026-02-22T21:52:42.175 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ error_code=124 2026-02-22T21:52:42.175 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set -e 2026-02-22T21:52:42.175 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' 124 -eq 124 ']' 2026-02-22T21:52:42.175 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:52:42.175 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T21:52:42.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_29 2026-02-22T21:52:42.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image_and_enable_mirror cluster2 mirror image_29 snapshot 128M 2026-02-22T21:52:42.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:52:42.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:52:42.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:52:42.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:52:42.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_29 2026-02-22T21:52:42.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:52:42.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T21:52:42.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n snapshot ']' 2026-02-22T21:52:42.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:52:42.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image cluster2 mirror image_29 128M 2026-02-22T21:52:42.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:52:42.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:52:42.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:52:42.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:52:42.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_29 2026-02-22T21:52:42.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:52:42.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local size=128 2026-02-22T21:52:42.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n 128M ']' 2026-02-22T21:52:42.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ size=128M 2026-02-22T21:52:42.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:52:42.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_29 2026-02-22T21:52:42.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T21:52:42.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:52:42.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' image = image ']' 2026-02-22T21:52:42.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:52:42.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ enable_mirror cluster2 mirror image_29 snapshot 2026-02-22T21:52:42.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:52:42.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:52:42.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_29 2026-02-22T21:52:42.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T21:52:42.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_29 snapshot 2026-02-22T21:52:43.152 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Mirroring enabled 2026-02-22T21:52:43.157 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 info mirror/image_29 2026-02-22T21:52:43.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_29': 2026-02-22T21:52:43.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 128 MiB in 32 objects 2026-02-22T21:52:43.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T21:52:43.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 1 2026-02-22T21:52:43.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 13a8992f7ac 2026-02-22T21:52:43.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.13a8992f7ac 2026-02-22T21:52:43.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T21:52:43.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock 2026-02-22T21:52:43.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T21:52:43.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T21:52:43.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:52:42 2026 2026-02-22T21:52:43.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:52:42 2026 2026-02-22T21:52:43.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:52:42 2026 2026-02-22T21:52:43.183 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T21:52:43.183 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T21:52:43.183 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: 199d7709-f322-4604-895e-a583a38cb8d5 2026-02-22T21:52:43.183 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: true 2026-02-22T21:52:43.183 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n '' ']' 2026-02-22T21:52:43.184 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ stress_write_image cluster2 mirror image_29 2026-02-22T21:52:43.184 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:52:43.184 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:52:43.184 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_29 2026-02-22T21:52:43.184 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-22T21:52:43.185 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local duration=10 2026-02-22T21:52:43.185 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set +e 2026-02-22T21:52:43.185 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ timeout 10s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_29 --debug-rbd=20 --debug-journaler=20 2026-02-22T21:52:43.256 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:100 IOs, 17016832 bytes 2026-02-22T21:52:43.288 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:200 IOs, 27990016 bytes 2026-02-22T21:52:43.333 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:300 IOs, 56229888 bytes 2026-02-22T21:52:43.376 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:400 IOs, 85382144 bytes 2026-02-22T21:52:43.431 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:500 IOs, 161602560 bytes 2026-02-22T21:52:43.469 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:600 IOs, 183179264 bytes 2026-02-22T21:52:43.531 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:700 IOs, 247701504 bytes 2026-02-22T21:52:43.574 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:800 IOs, 265647104 bytes 2026-02-22T21:52:43.620 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:900 IOs, 295365632 bytes 2026-02-22T21:52:43.647 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1000 IOs, 295994368 bytes 2026-02-22T21:52:43.697 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1100 IOs, 340816896 bytes 2026-02-22T21:52:43.739 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1200 IOs, 364972032 bytes 2026-02-22T21:52:43.791 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1300 IOs, 411599872 bytes 2026-02-22T21:52:43.882 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1400 IOs, 541434880 bytes 2026-02-22T21:52:43.932 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1500 IOs, 593108992 bytes 2026-02-22T21:52:43.965 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1600 IOs, 609412096 bytes 2026-02-22T21:52:44.009 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1700 IOs, 640821248 bytes 2026-02-22T21:52:44.057 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1800 IOs, 676870144 bytes 2026-02-22T21:52:44.092 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1900 IOs, 689844224 bytes 2026-02-22T21:52:44.130 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2000 IOs, 722414592 bytes 2026-02-22T21:52:44.188 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2100 IOs, 758300672 bytes 2026-02-22T21:52:44.240 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2200 IOs, 814121984 bytes 2026-02-22T21:52:44.298 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2300 IOs, 901192704 bytes 2026-02-22T21:52:44.382 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2400 IOs, 992395264 bytes 2026-02-22T21:52:44.468 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2500 IOs, 1098438656 bytes 2026-02-22T21:52:44.528 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2600 IOs, 1159373824 bytes 2026-02-22T21:52:44.583 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2700 IOs, 1192822784 bytes 2026-02-22T21:52:44.623 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2800 IOs, 1215108096 bytes 2026-02-22T21:52:44.650 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2900 IOs, 1215904768 bytes 2026-02-22T21:52:44.696 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3000 IOs, 1251469312 bytes 2026-02-22T21:52:44.752 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3100 IOs, 1299948544 bytes 2026-02-22T21:52:44.790 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3200 IOs, 1322893312 bytes 2026-02-22T21:52:44.850 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3300 IOs, 1411974144 bytes 2026-02-22T21:52:44.924 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3400 IOs, 1498170368 bytes 2026-02-22T21:52:44.950 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3500 IOs, 1498750976 bytes 2026-02-22T21:52:45.005 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3600 IOs, 1553487872 bytes 2026-02-22T21:52:45.063 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3700 IOs, 1624979456 bytes 2026-02-22T21:52:45.096 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3800 IOs, 1632758784 bytes 2026-02-22T21:52:45.126 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3900 IOs, 1640411136 bytes 2026-02-22T21:52:45.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4000 IOs, 1707417600 bytes 2026-02-22T21:52:45.222 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4100 IOs, 1710163968 bytes 2026-02-22T21:52:45.262 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4200 IOs, 1738624000 bytes 2026-02-22T21:52:45.318 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4300 IOs, 1811143680 bytes 2026-02-22T21:52:45.356 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4400 IOs, 1831414784 bytes 2026-02-22T21:52:45.398 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4500 IOs, 1855174656 bytes 2026-02-22T21:52:45.452 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4600 IOs, 1900812288 bytes 2026-02-22T21:52:45.491 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4700 IOs, 1924798464 bytes 2026-02-22T21:52:45.540 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4800 IOs, 1968655360 bytes 2026-02-22T21:52:45.603 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4900 IOs, 2023147520 bytes 2026-02-22T21:52:45.637 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5000 IOs, 2038851584 bytes 2026-02-22T21:52:45.697 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5100 IOs, 2106704896 bytes 2026-02-22T21:52:45.763 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5200 IOs, 2191030272 bytes 2026-02-22T21:52:45.792 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5300 IOs, 2191640576 bytes 2026-02-22T21:52:45.823 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5400 IOs, 2202082304 bytes 2026-02-22T21:52:45.872 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5500 IOs, 2227690496 bytes 2026-02-22T21:52:45.914 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5600 IOs, 2261487616 bytes 2026-02-22T21:52:45.969 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5700 IOs, 2307555328 bytes 2026-02-22T21:52:46.025 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5800 IOs, 2364829696 bytes 2026-02-22T21:52:46.059 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5900 IOs, 2381433856 bytes 2026-02-22T21:52:46.099 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6000 IOs, 2420847616 bytes 2026-02-22T21:52:46.160 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6100 IOs, 2486260736 bytes 2026-02-22T21:52:46.228 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6200 IOs, 2542338048 bytes 2026-02-22T21:52:46.274 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6300 IOs, 2576073728 bytes 2026-02-22T21:52:46.319 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6400 IOs, 2614910976 bytes 2026-02-22T21:52:46.379 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6500 IOs, 2675869696 bytes 2026-02-22T21:52:46.427 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6600 IOs, 2713723904 bytes 2026-02-22T21:52:46.465 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6700 IOs, 2733858816 bytes 2026-02-22T21:52:46.513 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6800 IOs, 2777011200 bytes 2026-02-22T21:52:46.551 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6900 IOs, 2799597568 bytes 2026-02-22T21:52:46.595 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7000 IOs, 2833822720 bytes 2026-02-22T21:52:46.647 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7100 IOs, 2872550400 bytes 2026-02-22T21:52:46.682 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7200 IOs, 2891346944 bytes 2026-02-22T21:52:46.750 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7300 IOs, 2967410688 bytes 2026-02-22T21:52:46.794 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7400 IOs, 3002891264 bytes 2026-02-22T21:52:46.850 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7500 IOs, 3058008064 bytes 2026-02-22T21:52:46.920 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7600 IOs, 3158996992 bytes 2026-02-22T21:52:46.962 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7700 IOs, 3191369728 bytes 2026-02-22T21:52:47.011 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7800 IOs, 3223746560 bytes 2026-02-22T21:52:47.047 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7900 IOs, 3239831552 bytes 2026-02-22T21:52:47.098 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8000 IOs, 3283432448 bytes 2026-02-22T21:52:47.158 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8100 IOs, 3351422976 bytes 2026-02-22T21:52:47.239 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8200 IOs, 3448071168 bytes 2026-02-22T21:52:47.288 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8300 IOs, 3485158400 bytes 2026-02-22T21:52:47.324 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8400 IOs, 3527706624 bytes 2026-02-22T21:52:47.375 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8500 IOs, 3547941888 bytes 2026-02-22T21:52:47.425 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8600 IOs, 3591527424 bytes 2026-02-22T21:52:47.465 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8700 IOs, 3610897408 bytes 2026-02-22T21:52:47.524 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8800 IOs, 3669249024 bytes 2026-02-22T21:52:47.563 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8900 IOs, 3681881088 bytes 2026-02-22T21:52:47.607 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9000 IOs, 3706392576 bytes 2026-02-22T21:52:47.666 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9100 IOs, 3781553152 bytes 2026-02-22T21:52:47.715 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9200 IOs, 3814531072 bytes 2026-02-22T21:52:47.753 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9300 IOs, 3833600000 bytes 2026-02-22T21:52:47.800 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9400 IOs, 3887107072 bytes 2026-02-22T21:52:47.842 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9500 IOs, 3899906048 bytes 2026-02-22T21:52:47.892 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9600 IOs, 3948358656 bytes 2026-02-22T21:52:47.959 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9700 IOs, 4025615360 bytes 2026-02-22T21:52:47.993 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9800 IOs, 4044879872 bytes 2026-02-22T21:52:48.051 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9900 IOs, 4119647232 bytes 2026-02-22T21:52:48.109 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10000 IOs, 4177513472 bytes 2026-02-22T21:52:48.158 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10100 IOs, 4221251584 bytes 2026-02-22T21:52:48.196 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10200 IOs, 4247711744 bytes 2026-02-22T21:52:48.236 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10300 IOs, 4262469632 bytes 2026-02-22T21:52:48.266 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10400 IOs, 4279854080 bytes 2026-02-22T21:52:48.340 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10500 IOs, 4351936512 bytes 2026-02-22T21:52:48.391 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10600 IOs, 4396603392 bytes 2026-02-22T21:52:48.429 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10700 IOs, 4434872320 bytes 2026-02-22T21:52:48.498 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10800 IOs, 4496142336 bytes 2026-02-22T21:52:48.557 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10900 IOs, 4571064320 bytes 2026-02-22T21:52:48.619 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11000 IOs, 4602471424 bytes 2026-02-22T21:52:48.676 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11100 IOs, 4664549376 bytes 2026-02-22T21:52:48.726 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11200 IOs, 4712285184 bytes 2026-02-22T21:52:48.765 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11300 IOs, 4723826688 bytes 2026-02-22T21:52:48.825 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11400 IOs, 4782488576 bytes 2026-02-22T21:52:48.884 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11500 IOs, 4858530816 bytes 2026-02-22T21:52:48.929 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11600 IOs, 4892781568 bytes 2026-02-22T21:52:48.977 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11700 IOs, 4949505024 bytes 2026-02-22T21:52:49.010 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11800 IOs, 4950137856 bytes 2026-02-22T21:52:49.053 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11900 IOs, 4988701696 bytes 2026-02-22T21:52:49.094 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12000 IOs, 5012118528 bytes 2026-02-22T21:52:49.132 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12100 IOs, 5036550144 bytes 2026-02-22T21:52:49.174 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12200 IOs, 5067963392 bytes 2026-02-22T21:52:49.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12300 IOs, 5074430976 bytes 2026-02-22T21:52:49.254 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12400 IOs, 5114577920 bytes 2026-02-22T21:52:49.323 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12500 IOs, 5213798400 bytes 2026-02-22T21:52:49.386 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12600 IOs, 5275076608 bytes 2026-02-22T21:52:49.429 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12700 IOs, 5308086272 bytes 2026-02-22T21:52:49.463 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12800 IOs, 5313141760 bytes 2026-02-22T21:52:49.497 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12900 IOs, 5316994048 bytes 2026-02-22T21:52:49.562 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13000 IOs, 5402285056 bytes 2026-02-22T21:52:49.617 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13100 IOs, 5450155008 bytes 2026-02-22T21:52:49.662 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13200 IOs, 5467858944 bytes 2026-02-22T21:52:49.706 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13300 IOs, 5507618816 bytes 2026-02-22T21:52:49.771 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13400 IOs, 5579145216 bytes 2026-02-22T21:52:49.808 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13500 IOs, 5591845888 bytes 2026-02-22T21:52:49.879 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13600 IOs, 5685929984 bytes 2026-02-22T21:52:49.918 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13700 IOs, 5711272960 bytes 2026-02-22T21:52:49.958 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13800 IOs, 5736960000 bytes 2026-02-22T21:52:49.986 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13900 IOs, 5742704640 bytes 2026-02-22T21:52:50.029 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14000 IOs, 5777147904 bytes 2026-02-22T21:52:50.056 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14100 IOs, 5777732608 bytes 2026-02-22T21:52:50.102 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14200 IOs, 5813653504 bytes 2026-02-22T21:52:50.152 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14300 IOs, 5874287616 bytes 2026-02-22T21:52:50.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14400 IOs, 5924857856 bytes 2026-02-22T21:52:50.262 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14500 IOs, 5967719424 bytes 2026-02-22T21:52:50.303 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14600 IOs, 6003251200 bytes 2026-02-22T21:52:50.356 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14700 IOs, 6051623936 bytes 2026-02-22T21:52:50.396 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14800 IOs, 6076723200 bytes 2026-02-22T21:52:50.426 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14900 IOs, 6087791616 bytes 2026-02-22T21:52:50.466 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15000 IOs, 6116566016 bytes 2026-02-22T21:52:50.513 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15100 IOs, 6140552192 bytes 2026-02-22T21:52:50.543 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15200 IOs, 6141163520 bytes 2026-02-22T21:52:50.589 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15300 IOs, 6198403072 bytes 2026-02-22T21:52:50.657 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15400 IOs, 6272445440 bytes 2026-02-22T21:52:50.712 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15500 IOs, 6317086720 bytes 2026-02-22T21:52:50.755 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15600 IOs, 6370627584 bytes 2026-02-22T21:52:50.808 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15700 IOs, 6408978432 bytes 2026-02-22T21:52:50.867 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15800 IOs, 6491162624 bytes 2026-02-22T21:52:50.918 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15900 IOs, 6526519296 bytes 2026-02-22T21:52:50.963 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16000 IOs, 6583563264 bytes 2026-02-22T21:52:51.006 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16100 IOs, 6593486848 bytes 2026-02-22T21:52:51.076 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16200 IOs, 6680096768 bytes 2026-02-22T21:52:51.117 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16300 IOs, 6704573440 bytes 2026-02-22T21:52:51.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16400 IOs, 6770808832 bytes 2026-02-22T21:52:51.246 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16500 IOs, 6858381312 bytes 2026-02-22T21:52:51.297 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16600 IOs, 6883837952 bytes 2026-02-22T21:52:51.346 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16700 IOs, 6934197248 bytes 2026-02-22T21:52:51.362 INFO:tasks.ceph.cluster2.osd.0.trial026.stderr:2026-02-22T21:52:51.361+0000 7ff4b543f640 -1 reset not still connected to 0x55b393a61520 2026-02-22T21:52:51.415 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16800 IOs, 6997516288 bytes 2026-02-22T21:52:51.457 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16900 IOs, 7027294208 bytes 2026-02-22T21:52:51.531 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17000 IOs, 7119059968 bytes 2026-02-22T21:52:51.582 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17100 IOs, 7163671552 bytes 2026-02-22T21:52:51.659 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17200 IOs, 7259916288 bytes 2026-02-22T21:52:51.694 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17300 IOs, 7260563456 bytes 2026-02-22T21:52:51.732 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17400 IOs, 7278902272 bytes 2026-02-22T21:52:51.796 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17500 IOs, 7348652032 bytes 2026-02-22T21:52:51.833 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17600 IOs, 7372410880 bytes 2026-02-22T21:52:51.898 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17700 IOs, 7434103808 bytes 2026-02-22T21:52:51.954 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17800 IOs, 7494678528 bytes 2026-02-22T21:52:52.002 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17900 IOs, 7538137088 bytes 2026-02-22T21:52:52.079 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18000 IOs, 7631717376 bytes 2026-02-22T21:52:52.129 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18100 IOs, 7670868992 bytes 2026-02-22T21:52:52.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18200 IOs, 7744624640 bytes 2026-02-22T21:52:52.240 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18300 IOs, 7773718528 bytes 2026-02-22T21:52:52.297 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18400 IOs, 7822822400 bytes 2026-02-22T21:52:52.342 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18500 IOs, 7858114560 bytes 2026-02-22T21:52:52.433 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18600 IOs, 7999318016 bytes 2026-02-22T21:52:52.494 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18700 IOs, 8041797632 bytes 2026-02-22T21:52:52.555 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18800 IOs, 8111569920 bytes 2026-02-22T21:52:52.589 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18900 IOs, 8129795072 bytes 2026-02-22T21:52:52.636 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19000 IOs, 8142099456 bytes 2026-02-22T21:52:52.677 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19100 IOs, 8159058944 bytes 2026-02-22T21:52:52.762 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19200 IOs, 8276883456 bytes 2026-02-22T21:52:52.808 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19300 IOs, 8308897792 bytes 2026-02-22T21:52:52.840 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19400 IOs, 8313558016 bytes 2026-02-22T21:52:52.897 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19500 IOs, 8368266240 bytes 2026-02-22T21:52:52.950 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19600 IOs, 8405948416 bytes 2026-02-22T21:52:52.994 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19700 IOs, 8434001920 bytes 2026-02-22T21:52:53.039 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19800 IOs, 8466488320 bytes 2026-02-22T21:52:53.092 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19900 IOs, 8503596032 bytes 2026-02-22T21:52:53.152 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20000 IOs, 8564348928 bytes 2026-02-22T21:52:53.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ error_code=124 2026-02-22T21:52:53.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set -e 2026-02-22T21:52:53.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' 124 -eq 124 ']' 2026-02-22T21:52:53.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:52:53.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T21:52:53.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_30 2026-02-22T21:52:53.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image_and_enable_mirror cluster2 mirror image_30 snapshot 128M 2026-02-22T21:52:53.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:52:53.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:52:53.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:52:53.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:52:53.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_30 2026-02-22T21:52:53.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:52:53.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T21:52:53.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n snapshot ']' 2026-02-22T21:52:53.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:52:53.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image cluster2 mirror image_30 128M 2026-02-22T21:52:53.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:52:53.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:52:53.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:52:53.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:52:53.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_30 2026-02-22T21:52:53.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:52:53.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local size=128 2026-02-22T21:52:53.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n 128M ']' 2026-02-22T21:52:53.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ size=128M 2026-02-22T21:52:53.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:52:53.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_30 2026-02-22T21:52:53.223 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T21:52:53.223 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:52:53.223 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' image = image ']' 2026-02-22T21:52:53.223 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:52:53.223 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ enable_mirror cluster2 mirror image_30 snapshot 2026-02-22T21:52:53.223 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:52:53.223 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:52:53.223 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_30 2026-02-22T21:52:53.223 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T21:52:53.223 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_30 snapshot 2026-02-22T21:52:54.160 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Mirroring enabled 2026-02-22T21:52:54.166 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 info mirror/image_30 2026-02-22T21:52:54.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_30': 2026-02-22T21:52:54.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 128 MiB in 32 objects 2026-02-22T21:52:54.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T21:52:54.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 1 2026-02-22T21:52:54.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 13b24e34ab20 2026-02-22T21:52:54.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.13b24e34ab20 2026-02-22T21:52:54.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T21:52:54.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock 2026-02-22T21:52:54.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T21:52:54.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T21:52:54.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:52:53 2026 2026-02-22T21:52:54.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:52:53 2026 2026-02-22T21:52:54.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:52:53 2026 2026-02-22T21:52:54.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T21:52:54.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T21:52:54.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: cce0d5f1-d366-4b28-ba1a-4d3fe01fd2e1 2026-02-22T21:52:54.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: true 2026-02-22T21:52:54.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n '' ']' 2026-02-22T21:52:54.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ stress_write_image cluster2 mirror image_30 2026-02-22T21:52:54.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:52:54.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:52:54.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_30 2026-02-22T21:52:54.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-22T21:52:54.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local duration=7 2026-02-22T21:52:54.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set +e 2026-02-22T21:52:54.195 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ timeout 7s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_30 --debug-rbd=20 --debug-journaler=20 2026-02-22T21:52:54.265 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:100 IOs, 19205120 bytes 2026-02-22T21:52:54.302 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:200 IOs, 43708416 bytes 2026-02-22T21:52:54.346 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:300 IOs, 83288064 bytes 2026-02-22T21:52:54.404 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:400 IOs, 144192512 bytes 2026-02-22T21:52:54.442 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:500 IOs, 161170432 bytes 2026-02-22T21:52:54.496 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:600 IOs, 230021120 bytes 2026-02-22T21:52:54.542 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:700 IOs, 249279488 bytes 2026-02-22T21:52:54.591 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:800 IOs, 287948800 bytes 2026-02-22T21:52:54.623 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:900 IOs, 293921792 bytes 2026-02-22T21:52:54.690 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1000 IOs, 347204608 bytes 2026-02-22T21:52:54.734 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1100 IOs, 380429312 bytes 2026-02-22T21:52:54.774 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1200 IOs, 408051712 bytes 2026-02-22T21:52:54.801 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1300 IOs, 408673280 bytes 2026-02-22T21:52:54.844 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1400 IOs, 439634944 bytes 2026-02-22T21:52:54.885 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1500 IOs, 458646528 bytes 2026-02-22T21:52:54.921 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1600 IOs, 479001600 bytes 2026-02-22T21:52:54.953 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1700 IOs, 479607808 bytes 2026-02-22T21:52:55.035 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1800 IOs, 572952576 bytes 2026-02-22T21:52:55.076 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1900 IOs, 599963648 bytes 2026-02-22T21:52:55.127 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2000 IOs, 642683904 bytes 2026-02-22T21:52:55.166 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2100 IOs, 660039680 bytes 2026-02-22T21:52:55.233 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2200 IOs, 736200704 bytes 2026-02-22T21:52:55.302 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2300 IOs, 797689856 bytes 2026-02-22T21:52:55.351 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2400 IOs, 836605952 bytes 2026-02-22T21:52:55.394 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2500 IOs, 861883392 bytes 2026-02-22T21:52:55.420 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2600 IOs, 862501888 bytes 2026-02-22T21:52:55.466 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2700 IOs, 909763584 bytes 2026-02-22T21:52:55.499 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2800 IOs, 910732288 bytes 2026-02-22T21:52:55.569 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2900 IOs, 981786624 bytes 2026-02-22T21:52:55.632 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3000 IOs, 1069037568 bytes 2026-02-22T21:52:55.703 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3100 IOs, 1120146432 bytes 2026-02-22T21:52:55.749 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3200 IOs, 1149519872 bytes 2026-02-22T21:52:55.791 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3300 IOs, 1185434624 bytes 2026-02-22T21:52:55.823 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3400 IOs, 1219554304 bytes 2026-02-22T21:52:55.901 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3500 IOs, 1310067712 bytes 2026-02-22T21:52:55.952 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3600 IOs, 1365800960 bytes 2026-02-22T21:52:55.993 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3700 IOs, 1387383808 bytes 2026-02-22T21:52:56.020 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3800 IOs, 1388001280 bytes 2026-02-22T21:52:56.092 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3900 IOs, 1476728832 bytes 2026-02-22T21:52:56.155 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4000 IOs, 1544358912 bytes 2026-02-22T21:52:56.237 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4100 IOs, 1657672704 bytes 2026-02-22T21:52:56.305 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4200 IOs, 1734685696 bytes 2026-02-22T21:52:56.358 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4300 IOs, 1789601792 bytes 2026-02-22T21:52:56.402 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4400 IOs, 1809329152 bytes 2026-02-22T21:52:56.442 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4500 IOs, 1834184704 bytes 2026-02-22T21:52:56.471 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4600 IOs, 1835618304 bytes 2026-02-22T21:52:56.508 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4700 IOs, 1849885696 bytes 2026-02-22T21:52:56.546 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4800 IOs, 1880963072 bytes 2026-02-22T21:52:56.618 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4900 IOs, 1938049024 bytes 2026-02-22T21:52:56.655 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5000 IOs, 1953723392 bytes 2026-02-22T21:52:56.694 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5100 IOs, 1981337600 bytes 2026-02-22T21:52:56.733 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5200 IOs, 2006408192 bytes 2026-02-22T21:52:56.797 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5300 IOs, 2093484032 bytes 2026-02-22T21:52:56.853 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5400 IOs, 2130572288 bytes 2026-02-22T21:52:56.885 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5500 IOs, 2134599680 bytes 2026-02-22T21:52:56.952 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5600 IOs, 2199464960 bytes 2026-02-22T21:52:57.017 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5700 IOs, 2277733376 bytes 2026-02-22T21:52:57.045 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5800 IOs, 2282242048 bytes 2026-02-22T21:52:57.095 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5900 IOs, 2330573824 bytes 2026-02-22T21:52:57.153 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6000 IOs, 2383500288 bytes 2026-02-22T21:52:57.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6100 IOs, 2457120768 bytes 2026-02-22T21:52:57.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6200 IOs, 2513096704 bytes 2026-02-22T21:52:57.320 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6300 IOs, 2535289856 bytes 2026-02-22T21:52:57.372 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6400 IOs, 2582211584 bytes 2026-02-22T21:52:57.632 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6500 IOs, 2645423104 bytes 2026-02-22T21:52:57.669 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6600 IOs, 2653368320 bytes 2026-02-22T21:52:57.722 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6700 IOs, 2695291904 bytes 2026-02-22T21:52:57.775 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6800 IOs, 2749949952 bytes 2026-02-22T21:52:57.820 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6900 IOs, 2780241920 bytes 2026-02-22T21:52:57.856 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7000 IOs, 2791941120 bytes 2026-02-22T21:52:57.911 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7100 IOs, 2874806272 bytes 2026-02-22T21:52:57.974 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7200 IOs, 2920064000 bytes 2026-02-22T21:52:58.009 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7300 IOs, 2934244352 bytes 2026-02-22T21:52:58.056 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7400 IOs, 2970400768 bytes 2026-02-22T21:52:58.099 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7500 IOs, 3003807744 bytes 2026-02-22T21:52:58.141 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7600 IOs, 3033055232 bytes 2026-02-22T21:52:58.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7700 IOs, 3055664128 bytes 2026-02-22T21:52:58.236 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7800 IOs, 3121828864 bytes 2026-02-22T21:52:58.301 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7900 IOs, 3189069824 bytes 2026-02-22T21:52:58.353 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8000 IOs, 3220660224 bytes 2026-02-22T21:52:58.423 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8100 IOs, 3299957760 bytes 2026-02-22T21:52:58.451 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8200 IOs, 3300587520 bytes 2026-02-22T21:52:58.514 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8300 IOs, 3379851264 bytes 2026-02-22T21:52:58.549 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8400 IOs, 3390520320 bytes 2026-02-22T21:52:58.637 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8500 IOs, 3477539840 bytes 2026-02-22T21:52:58.675 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8600 IOs, 3496121344 bytes 2026-02-22T21:52:58.709 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8700 IOs, 3502417920 bytes 2026-02-22T21:52:58.743 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8800 IOs, 3512864768 bytes 2026-02-22T21:52:58.777 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8900 IOs, 3525795840 bytes 2026-02-22T21:52:58.829 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9000 IOs, 3567422464 bytes 2026-02-22T21:52:58.870 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9100 IOs, 3603236864 bytes 2026-02-22T21:52:58.914 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9200 IOs, 3633855488 bytes 2026-02-22T21:52:58.958 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9300 IOs, 3649233920 bytes 2026-02-22T21:52:59.001 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9400 IOs, 3677387776 bytes 2026-02-22T21:52:59.040 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9500 IOs, 3691458560 bytes 2026-02-22T21:52:59.084 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9600 IOs, 3721905152 bytes 2026-02-22T21:52:59.117 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9700 IOs, 3743017984 bytes 2026-02-22T21:52:59.186 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9800 IOs, 3817834496 bytes 2026-02-22T21:52:59.219 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9900 IOs, 3838625792 bytes 2026-02-22T21:52:59.274 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10000 IOs, 3895271424 bytes 2026-02-22T21:52:59.340 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10100 IOs, 3971457024 bytes 2026-02-22T21:52:59.381 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10200 IOs, 3992861696 bytes 2026-02-22T21:52:59.416 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10300 IOs, 4019678208 bytes 2026-02-22T21:52:59.494 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10400 IOs, 4111634432 bytes 2026-02-22T21:52:59.545 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10500 IOs, 4180197376 bytes 2026-02-22T21:52:59.610 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10600 IOs, 4221367296 bytes 2026-02-22T21:52:59.669 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10700 IOs, 4271756288 bytes 2026-02-22T21:52:59.727 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10800 IOs, 4321317888 bytes 2026-02-22T21:52:59.779 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10900 IOs, 4371662848 bytes 2026-02-22T21:52:59.840 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11000 IOs, 4421721088 bytes 2026-02-22T21:52:59.908 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11100 IOs, 4509572096 bytes 2026-02-22T21:52:59.984 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11200 IOs, 4609369088 bytes 2026-02-22T21:53:00.038 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11300 IOs, 4662387712 bytes 2026-02-22T21:53:00.133 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11400 IOs, 4791841792 bytes 2026-02-22T21:53:00.188 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11500 IOs, 4852340736 bytes 2026-02-22T21:53:00.273 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11600 IOs, 4958994432 bytes 2026-02-22T21:53:00.339 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11700 IOs, 5031857152 bytes 2026-02-22T21:53:00.381 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11800 IOs, 5055910912 bytes 2026-02-22T21:53:00.414 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11900 IOs, 5062524928 bytes 2026-02-22T21:53:00.460 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12000 IOs, 5101367296 bytes 2026-02-22T21:53:00.499 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12100 IOs, 5114892288 bytes 2026-02-22T21:53:00.561 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12200 IOs, 5187585024 bytes 2026-02-22T21:53:00.621 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12300 IOs, 5244224512 bytes 2026-02-22T21:53:00.682 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12400 IOs, 5304648704 bytes 2026-02-22T21:53:00.721 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12500 IOs, 5338283008 bytes 2026-02-22T21:53:00.764 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12600 IOs, 5366601728 bytes 2026-02-22T21:53:00.815 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12700 IOs, 5396839424 bytes 2026-02-22T21:53:00.846 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12800 IOs, 5403075584 bytes 2026-02-22T21:53:00.907 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12900 IOs, 5469847552 bytes 2026-02-22T21:53:00.944 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13000 IOs, 5487032320 bytes 2026-02-22T21:53:01.013 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13100 IOs, 5565835264 bytes 2026-02-22T21:53:01.073 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13200 IOs, 5617018880 bytes 2026-02-22T21:53:01.148 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13300 IOs, 5728367616 bytes 2026-02-22T21:53:01.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13400 IOs, 5743966208 bytes 2026-02-22T21:53:01.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ error_code=124 2026-02-22T21:53:01.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set -e 2026-02-22T21:53:01.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' 124 -eq 124 ']' 2026-02-22T21:53:01.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:53:01.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T21:53:01.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_31 2026-02-22T21:53:01.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image_and_enable_mirror cluster2 mirror image_31 snapshot 128M 2026-02-22T21:53:01.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:53:01.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:53:01.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:53:01.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:53:01.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_31 2026-02-22T21:53:01.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:53:01.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T21:53:01.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n snapshot ']' 2026-02-22T21:53:01.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:53:01.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image cluster2 mirror image_31 128M 2026-02-22T21:53:01.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:53:01.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:53:01.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:53:01.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:53:01.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_31 2026-02-22T21:53:01.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:53:01.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local size=128 2026-02-22T21:53:01.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n 128M ']' 2026-02-22T21:53:01.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ size=128M 2026-02-22T21:53:01.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:53:01.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_31 2026-02-22T21:53:01.230 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T21:53:01.230 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:53:01.231 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' image = image ']' 2026-02-22T21:53:01.231 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:53:01.231 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ enable_mirror cluster2 mirror image_31 snapshot 2026-02-22T21:53:01.231 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:53:01.231 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:53:01.231 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_31 2026-02-22T21:53:01.231 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T21:53:01.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_31 snapshot 2026-02-22T21:53:02.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Mirroring enabled 2026-02-22T21:53:02.169 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 info mirror/image_31 2026-02-22T21:53:02.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_31': 2026-02-22T21:53:02.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 128 MiB in 32 objects 2026-02-22T21:53:02.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T21:53:02.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 1 2026-02-22T21:53:02.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 13bccceb269a 2026-02-22T21:53:02.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.13bccceb269a 2026-02-22T21:53:02.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T21:53:02.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock 2026-02-22T21:53:02.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T21:53:02.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T21:53:02.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:53:01 2026 2026-02-22T21:53:02.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:53:01 2026 2026-02-22T21:53:02.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:53:01 2026 2026-02-22T21:53:02.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T21:53:02.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T21:53:02.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: 97a4414e-20e7-4288-9c80-0dce42f41d4f 2026-02-22T21:53:02.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: true 2026-02-22T21:53:02.195 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n '' ']' 2026-02-22T21:53:02.195 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ stress_write_image cluster2 mirror image_31 2026-02-22T21:53:02.195 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:53:02.195 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:53:02.195 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_31 2026-02-22T21:53:02.195 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-22T21:53:02.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local duration=6 2026-02-22T21:53:02.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set +e 2026-02-22T21:53:02.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ timeout 6s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_31 --debug-rbd=20 --debug-journaler=20 2026-02-22T21:53:02.282 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:100 IOs, 36959232 bytes 2026-02-22T21:53:02.341 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:200 IOs, 92265472 bytes 2026-02-22T21:53:02.400 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:300 IOs, 152176640 bytes 2026-02-22T21:53:02.446 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:400 IOs, 195492864 bytes 2026-02-22T21:53:02.506 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:500 IOs, 259355648 bytes 2026-02-22T21:53:02.571 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:600 IOs, 316806144 bytes 2026-02-22T21:53:02.614 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:700 IOs, 342142976 bytes 2026-02-22T21:53:02.659 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:800 IOs, 368357376 bytes 2026-02-22T21:53:02.707 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:900 IOs, 407379968 bytes 2026-02-22T21:53:02.748 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1000 IOs, 432579584 bytes 2026-02-22T21:53:02.806 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1100 IOs, 493576192 bytes 2026-02-22T21:53:02.844 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1200 IOs, 506779648 bytes 2026-02-22T21:53:02.911 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1300 IOs, 588777472 bytes 2026-02-22T21:53:02.963 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1400 IOs, 634198016 bytes 2026-02-22T21:53:03.001 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1500 IOs, 659654656 bytes 2026-02-22T21:53:03.046 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1600 IOs, 679358464 bytes 2026-02-22T21:53:03.085 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1700 IOs, 702223360 bytes 2026-02-22T21:53:03.142 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1800 IOs, 760051712 bytes 2026-02-22T21:53:03.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1900 IOs, 816036864 bytes 2026-02-22T21:53:03.255 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2000 IOs, 861670400 bytes 2026-02-22T21:53:03.307 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2100 IOs, 897085440 bytes 2026-02-22T21:53:03.364 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2200 IOs, 941276160 bytes 2026-02-22T21:53:03.403 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2300 IOs, 980603904 bytes 2026-02-22T21:53:03.454 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2400 IOs, 1003277312 bytes 2026-02-22T21:53:03.700 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2500 IOs, 1041475584 bytes 2026-02-22T21:53:03.752 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2600 IOs, 1073744896 bytes 2026-02-22T21:53:03.805 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2700 IOs, 1139676160 bytes 2026-02-22T21:53:03.858 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2800 IOs, 1175318528 bytes 2026-02-22T21:53:03.899 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2900 IOs, 1187128320 bytes 2026-02-22T21:53:03.964 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3000 IOs, 1258144768 bytes 2026-02-22T21:53:03.999 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3100 IOs, 1281153024 bytes 2026-02-22T21:53:04.040 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3200 IOs, 1317388288 bytes 2026-02-22T21:53:04.099 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3300 IOs, 1381858304 bytes 2026-02-22T21:53:04.170 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3400 IOs, 1433864192 bytes 2026-02-22T21:53:04.215 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3500 IOs, 1465290752 bytes 2026-02-22T21:53:04.260 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3600 IOs, 1499809792 bytes 2026-02-22T21:53:04.322 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3700 IOs, 1563492352 bytes 2026-02-22T21:53:04.352 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3800 IOs, 1567497216 bytes 2026-02-22T21:53:04.395 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3900 IOs, 1600729088 bytes 2026-02-22T21:53:04.449 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4000 IOs, 1648504832 bytes 2026-02-22T21:53:04.476 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4100 IOs, 1649114112 bytes 2026-02-22T21:53:04.541 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4200 IOs, 1724066816 bytes 2026-02-22T21:53:04.583 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4300 IOs, 1746513920 bytes 2026-02-22T21:53:04.648 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4400 IOs, 1811296256 bytes 2026-02-22T21:53:04.692 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4500 IOs, 1828912128 bytes 2026-02-22T21:53:04.728 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4600 IOs, 1846105088 bytes 2026-02-22T21:53:04.786 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4700 IOs, 1901415424 bytes 2026-02-22T21:53:04.840 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4800 IOs, 1947712512 bytes 2026-02-22T21:53:04.889 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4900 IOs, 1986296832 bytes 2026-02-22T21:53:04.960 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5000 IOs, 2067254272 bytes 2026-02-22T21:53:05.020 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5100 IOs, 2149433344 bytes 2026-02-22T21:53:05.082 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5200 IOs, 2192010240 bytes 2026-02-22T21:53:05.145 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5300 IOs, 2263118848 bytes 2026-02-22T21:53:05.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5400 IOs, 2304369664 bytes 2026-02-22T21:53:05.255 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5500 IOs, 2368189440 bytes 2026-02-22T21:53:05.311 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5600 IOs, 2435791872 bytes 2026-02-22T21:53:05.354 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5700 IOs, 2459303936 bytes 2026-02-22T21:53:05.417 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5800 IOs, 2514598912 bytes 2026-02-22T21:53:05.459 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5900 IOs, 2536672256 bytes 2026-02-22T21:53:05.524 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6000 IOs, 2600221696 bytes 2026-02-22T21:53:05.590 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6100 IOs, 2671405056 bytes 2026-02-22T21:53:05.643 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6200 IOs, 2721487872 bytes 2026-02-22T21:53:05.692 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6300 IOs, 2743747584 bytes 2026-02-22T21:53:05.751 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6400 IOs, 2790533120 bytes 2026-02-22T21:53:05.813 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6500 IOs, 2853812224 bytes 2026-02-22T21:53:05.858 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6600 IOs, 2885214208 bytes 2026-02-22T21:53:05.916 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6700 IOs, 2942268416 bytes 2026-02-22T21:53:05.951 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6800 IOs, 2959410176 bytes 2026-02-22T21:53:05.983 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6900 IOs, 2962443264 bytes 2026-02-22T21:53:06.049 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7000 IOs, 3035001856 bytes 2026-02-22T21:53:06.126 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7100 IOs, 3144193024 bytes 2026-02-22T21:53:06.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7200 IOs, 3223374848 bytes 2026-02-22T21:53:06.252 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7300 IOs, 3258029056 bytes 2026-02-22T21:53:06.306 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7400 IOs, 3301842944 bytes 2026-02-22T21:53:06.336 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7500 IOs, 3304275968 bytes 2026-02-22T21:53:06.379 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7600 IOs, 3342869504 bytes 2026-02-22T21:53:06.445 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7700 IOs, 3403438080 bytes 2026-02-22T21:53:06.484 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7800 IOs, 3423176704 bytes 2026-02-22T21:53:06.530 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7900 IOs, 3474856960 bytes 2026-02-22T21:53:06.589 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8000 IOs, 3527603200 bytes 2026-02-22T21:53:06.637 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8100 IOs, 3557427200 bytes 2026-02-22T21:53:06.690 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8200 IOs, 3579036672 bytes 2026-02-22T21:53:06.733 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8300 IOs, 3600899072 bytes 2026-02-22T21:53:06.763 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8400 IOs, 3601517568 bytes 2026-02-22T21:53:06.821 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8500 IOs, 3670923264 bytes 2026-02-22T21:53:06.858 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8600 IOs, 3688544256 bytes 2026-02-22T21:53:06.897 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8700 IOs, 3700807680 bytes 2026-02-22T21:53:06.939 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8800 IOs, 3722027008 bytes 2026-02-22T21:53:07.008 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8900 IOs, 3802687488 bytes 2026-02-22T21:53:07.066 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9000 IOs, 3864555520 bytes 2026-02-22T21:53:07.101 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9100 IOs, 3876905984 bytes 2026-02-22T21:53:07.139 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9200 IOs, 3888614400 bytes 2026-02-22T21:53:07.151 INFO:tasks.ceph.cluster2.osd.1.trial026.stderr:2026-02-22T21:53:07.150+0000 7f82fc108640 -1 reset not still connected to 0x55beaba61040 2026-02-22T21:53:07.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9300 IOs, 3991494656 bytes 2026-02-22T21:53:07.266 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9400 IOs, 4024189952 bytes 2026-02-22T21:53:07.306 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9500 IOs, 4047525888 bytes 2026-02-22T21:53:07.360 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9600 IOs, 4103763968 bytes 2026-02-22T21:53:07.409 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9700 IOs, 4136937472 bytes 2026-02-22T21:53:07.460 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9800 IOs, 4186342400 bytes 2026-02-22T21:53:07.508 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9900 IOs, 4208888832 bytes 2026-02-22T21:53:07.560 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10000 IOs, 4261988352 bytes 2026-02-22T21:53:07.610 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10100 IOs, 4308657152 bytes 2026-02-22T21:53:07.644 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10200 IOs, 4319643648 bytes 2026-02-22T21:53:07.723 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10300 IOs, 4402313216 bytes 2026-02-22T21:53:07.771 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10400 IOs, 4425906176 bytes 2026-02-22T21:53:07.815 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10500 IOs, 4457090048 bytes 2026-02-22T21:53:07.862 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10600 IOs, 4504275968 bytes 2026-02-22T21:53:07.930 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10700 IOs, 4575761408 bytes 2026-02-22T21:53:07.973 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10800 IOs, 4603625472 bytes 2026-02-22T21:53:08.020 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10900 IOs, 4630194176 bytes 2026-02-22T21:53:08.077 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11000 IOs, 4685666304 bytes 2026-02-22T21:53:08.138 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11100 IOs, 4760083456 bytes 2026-02-22T21:53:08.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ error_code=124 2026-02-22T21:53:08.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set -e 2026-02-22T21:53:08.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' 124 -eq 124 ']' 2026-02-22T21:53:08.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:53:08.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T21:53:08.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_32 2026-02-22T21:53:08.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image_and_enable_mirror cluster2 mirror image_32 snapshot 128M 2026-02-22T21:53:08.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:53:08.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:53:08.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:53:08.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:53:08.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_32 2026-02-22T21:53:08.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:53:08.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T21:53:08.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n snapshot ']' 2026-02-22T21:53:08.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:53:08.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image cluster2 mirror image_32 128M 2026-02-22T21:53:08.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:53:08.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:53:08.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:53:08.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:53:08.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_32 2026-02-22T21:53:08.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:53:08.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local size=128 2026-02-22T21:53:08.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n 128M ']' 2026-02-22T21:53:08.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ size=128M 2026-02-22T21:53:08.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:53:08.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_32 2026-02-22T21:53:08.236 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T21:53:08.236 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:53:08.236 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' image = image ']' 2026-02-22T21:53:08.236 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:53:08.236 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ enable_mirror cluster2 mirror image_32 snapshot 2026-02-22T21:53:08.236 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:53:08.236 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:53:08.236 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_32 2026-02-22T21:53:08.236 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T21:53:08.236 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_32 snapshot 2026-02-22T21:53:09.124 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Mirroring enabled 2026-02-22T21:53:09.129 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 info mirror/image_32 2026-02-22T21:53:09.153 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_32': 2026-02-22T21:53:09.154 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 128 MiB in 32 objects 2026-02-22T21:53:09.154 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T21:53:09.154 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 1 2026-02-22T21:53:09.154 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 13c4196b2937 2026-02-22T21:53:09.154 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.13c4196b2937 2026-02-22T21:53:09.154 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T21:53:09.154 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock 2026-02-22T21:53:09.154 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T21:53:09.154 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T21:53:09.154 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:53:08 2026 2026-02-22T21:53:09.155 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:53:08 2026 2026-02-22T21:53:09.155 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:53:08 2026 2026-02-22T21:53:09.155 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T21:53:09.155 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T21:53:09.155 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: 533da63c-6a64-4a63-a224-d042b1c3553d 2026-02-22T21:53:09.155 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: true 2026-02-22T21:53:09.156 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n '' ']' 2026-02-22T21:53:09.156 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ stress_write_image cluster2 mirror image_32 2026-02-22T21:53:09.156 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:53:09.156 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:53:09.156 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_32 2026-02-22T21:53:09.157 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-22T21:53:09.157 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local duration=9 2026-02-22T21:53:09.157 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set +e 2026-02-22T21:53:09.157 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ timeout 9s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_32 --debug-rbd=20 --debug-journaler=20 2026-02-22T21:53:09.255 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:100 IOs, 67136512 bytes 2026-02-22T21:53:09.308 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:200 IOs, 109636608 bytes 2026-02-22T21:53:09.372 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:300 IOs, 188506112 bytes 2026-02-22T21:53:09.425 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:400 IOs, 231410688 bytes 2026-02-22T21:53:09.484 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:500 IOs, 295192576 bytes 2026-02-22T21:53:09.541 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:600 IOs, 346594304 bytes 2026-02-22T21:53:09.585 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:700 IOs, 375676928 bytes 2026-02-22T21:53:09.615 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:800 IOs, 382084096 bytes 2026-02-22T21:53:09.684 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:900 IOs, 487429120 bytes 2026-02-22T21:53:09.753 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1000 IOs, 538960896 bytes 2026-02-22T21:53:09.784 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1100 IOs, 539575296 bytes 2026-02-22T21:53:09.819 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1200 IOs, 557941760 bytes 2026-02-22T21:53:09.876 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1300 IOs, 608117760 bytes 2026-02-22T21:53:09.926 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1400 IOs, 648883200 bytes 2026-02-22T21:53:09.979 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1500 IOs, 700922880 bytes 2026-02-22T21:53:10.028 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1600 IOs, 736824320 bytes 2026-02-22T21:53:10.077 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1700 IOs, 780682240 bytes 2026-02-22T21:53:10.117 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1800 IOs, 802856960 bytes 2026-02-22T21:53:10.158 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1900 IOs, 824049664 bytes 2026-02-22T21:53:10.236 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2000 IOs, 917452800 bytes 2026-02-22T21:53:10.277 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2100 IOs, 943048704 bytes 2026-02-22T21:53:10.324 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2200 IOs, 973437952 bytes 2026-02-22T21:53:10.365 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2300 IOs, 1004512256 bytes 2026-02-22T21:53:10.429 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2400 IOs, 1064182784 bytes 2026-02-22T21:53:10.479 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2500 IOs, 1095507968 bytes 2026-02-22T21:53:10.543 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2600 IOs, 1156343808 bytes 2026-02-22T21:53:10.591 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2700 IOs, 1204400128 bytes 2026-02-22T21:53:10.660 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2800 IOs, 1284028416 bytes 2026-02-22T21:53:10.697 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2900 IOs, 1301168128 bytes 2026-02-22T21:53:10.738 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3000 IOs, 1318286336 bytes 2026-02-22T21:53:10.780 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3100 IOs, 1337816064 bytes 2026-02-22T21:53:10.807 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3200 IOs, 1338408960 bytes 2026-02-22T21:53:10.843 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3300 IOs, 1356877824 bytes 2026-02-22T21:53:10.905 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3400 IOs, 1432674304 bytes 2026-02-22T21:53:10.978 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3500 IOs, 1526365184 bytes 2026-02-22T21:53:11.051 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3600 IOs, 1600733184 bytes 2026-02-22T21:53:11.119 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3700 IOs, 1676827648 bytes 2026-02-22T21:53:11.187 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3800 IOs, 1746703360 bytes 2026-02-22T21:53:11.241 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3900 IOs, 1802550272 bytes 2026-02-22T21:53:11.297 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4000 IOs, 1841308672 bytes 2026-02-22T21:53:11.346 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4100 IOs, 1882915840 bytes 2026-02-22T21:53:11.400 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4200 IOs, 1925046272 bytes 2026-02-22T21:53:11.442 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4300 IOs, 1944491008 bytes 2026-02-22T21:53:11.491 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4400 IOs, 1993925632 bytes 2026-02-22T21:53:11.535 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4500 IOs, 2022915072 bytes 2026-02-22T21:53:11.591 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4600 IOs, 2072110080 bytes 2026-02-22T21:53:11.622 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4700 IOs, 2080432128 bytes 2026-02-22T21:53:11.658 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4800 IOs, 2095248384 bytes 2026-02-22T21:53:11.708 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4900 IOs, 2137459712 bytes 2026-02-22T21:53:11.755 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5000 IOs, 2168530944 bytes 2026-02-22T21:53:11.825 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5100 IOs, 2268424192 bytes 2026-02-22T21:53:11.898 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5200 IOs, 2345426944 bytes 2026-02-22T21:53:11.958 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5300 IOs, 2384032768 bytes 2026-02-22T21:53:11.996 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5400 IOs, 2406037504 bytes 2026-02-22T21:53:12.061 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5500 IOs, 2487984128 bytes 2026-02-22T21:53:12.128 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5600 IOs, 2557856768 bytes 2026-02-22T21:53:12.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5700 IOs, 2594462720 bytes 2026-02-22T21:53:12.234 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5800 IOs, 2665571328 bytes 2026-02-22T21:53:12.302 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5900 IOs, 2723690496 bytes 2026-02-22T21:53:12.371 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6000 IOs, 2803501056 bytes 2026-02-22T21:53:12.452 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6100 IOs, 2887331840 bytes 2026-02-22T21:53:12.498 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6200 IOs, 2924489728 bytes 2026-02-22T21:53:12.582 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6300 IOs, 3033182208 bytes 2026-02-22T21:53:12.623 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6400 IOs, 3067465728 bytes 2026-02-22T21:53:12.665 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6500 IOs, 3094351872 bytes 2026-02-22T21:53:12.707 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6600 IOs, 3132253184 bytes 2026-02-22T21:53:12.752 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6700 IOs, 3171622912 bytes 2026-02-22T21:53:12.827 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6800 IOs, 3272036352 bytes 2026-02-22T21:53:12.885 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6900 IOs, 3320111104 bytes 2026-02-22T21:53:12.937 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7000 IOs, 3372343296 bytes 2026-02-22T21:53:12.981 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7100 IOs, 3405806592 bytes 2026-02-22T21:53:13.062 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7200 IOs, 3506625536 bytes 2026-02-22T21:53:13.146 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7300 IOs, 3602617344 bytes 2026-02-22T21:53:13.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7400 IOs, 3677209600 bytes 2026-02-22T21:53:13.283 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7500 IOs, 3782356992 bytes 2026-02-22T21:53:13.336 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7600 IOs, 3813387264 bytes 2026-02-22T21:53:13.399 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7700 IOs, 3879629824 bytes 2026-02-22T21:53:13.445 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7800 IOs, 3909072896 bytes 2026-02-22T21:53:13.491 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7900 IOs, 3929333760 bytes 2026-02-22T21:53:13.531 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8000 IOs, 3967939584 bytes 2026-02-22T21:53:13.581 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8100 IOs, 3992882176 bytes 2026-02-22T21:53:13.623 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8200 IOs, 4020660224 bytes 2026-02-22T21:53:13.651 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8300 IOs, 4024811520 bytes 2026-02-22T21:53:13.695 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8400 IOs, 4061214720 bytes 2026-02-22T21:53:13.740 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8500 IOs, 4084677632 bytes 2026-02-22T21:53:13.797 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8600 IOs, 4137582592 bytes 2026-02-22T21:53:13.857 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8700 IOs, 4197916672 bytes 2026-02-22T21:53:13.897 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8800 IOs, 4217932800 bytes 2026-02-22T21:53:13.933 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8900 IOs, 4232073216 bytes 2026-02-22T21:53:13.968 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9000 IOs, 4244531200 bytes 2026-02-22T21:53:14.042 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9100 IOs, 4328363008 bytes 2026-02-22T21:53:14.086 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9200 IOs, 4347716608 bytes 2026-02-22T21:53:14.126 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9300 IOs, 4368491520 bytes 2026-02-22T21:53:14.173 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.7581372327778786) > weight (0.25) 2026-02-22T21:53:14.174 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.46982684933510854) > weight (0.25) 2026-02-22T21:53:14.174 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.339130327367102) > weight (0.25) 2026-02-22T21:53:14.174 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.3 2026-02-22T21:53:14.174 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Sent signal 15 2026-02-22T21:53:14.174 INFO:tasks.rbd_mirror_thrash:waiting for 9.611725905131374 secs before reviving daemons 2026-02-22T21:53:14.175 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial026.stderr:2026-02-22T21:53:14.174+0000 7fcde616c640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.3 (PID: 48244) UID: 1000 2026-02-22T21:53:14.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9400 IOs, 4455490560 bytes 2026-02-22T21:53:14.279 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9500 IOs, 4540304384 bytes 2026-02-22T21:53:14.326 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9600 IOs, 4573975552 bytes 2026-02-22T21:53:14.373 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9700 IOs, 4614970368 bytes 2026-02-22T21:53:14.436 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9800 IOs, 4671816704 bytes 2026-02-22T21:53:14.490 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9900 IOs, 4710312960 bytes 2026-02-22T21:53:14.566 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10000 IOs, 4820384768 bytes 2026-02-22T21:53:14.607 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10100 IOs, 4852342784 bytes 2026-02-22T21:53:14.687 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10200 IOs, 4931793920 bytes 2026-02-22T21:53:14.731 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10300 IOs, 4960154624 bytes 2026-02-22T21:53:14.774 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10400 IOs, 4977581056 bytes 2026-02-22T21:53:14.822 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10500 IOs, 5005142016 bytes 2026-02-22T21:53:14.877 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10600 IOs, 5071886336 bytes 2026-02-22T21:53:14.945 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10700 IOs, 5125230592 bytes 2026-02-22T21:53:15.019 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10800 IOs, 5228370944 bytes 2026-02-22T21:53:15.059 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10900 IOs, 5260685312 bytes 2026-02-22T21:53:15.112 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11000 IOs, 5295451136 bytes 2026-02-22T21:53:15.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11100 IOs, 5356461056 bytes 2026-02-22T21:53:15.234 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11200 IOs, 5428099072 bytes 2026-02-22T21:53:15.264 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11300 IOs, 5456069632 bytes 2026-02-22T21:53:15.305 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11400 IOs, 5479862272 bytes 2026-02-22T21:53:15.348 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11500 IOs, 5515532288 bytes 2026-02-22T21:53:15.413 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11600 IOs, 5583936512 bytes 2026-02-22T21:53:15.456 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11700 IOs, 5609763840 bytes 2026-02-22T21:53:15.531 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11800 IOs, 5673738240 bytes 2026-02-22T21:53:15.569 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11900 IOs, 5692496896 bytes 2026-02-22T21:53:15.629 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12000 IOs, 5758429184 bytes 2026-02-22T21:53:15.698 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12100 IOs, 5848429568 bytes 2026-02-22T21:53:15.733 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12200 IOs, 5865886720 bytes 2026-02-22T21:53:15.776 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12300 IOs, 5892524032 bytes 2026-02-22T21:53:15.828 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12400 IOs, 5920448512 bytes 2026-02-22T21:53:15.860 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12500 IOs, 5926870016 bytes 2026-02-22T21:53:15.920 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12600 IOs, 5986882560 bytes 2026-02-22T21:53:15.971 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12700 IOs, 6030503936 bytes 2026-02-22T21:53:16.014 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12800 IOs, 6062928896 bytes 2026-02-22T21:53:16.069 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12900 IOs, 6108651520 bytes 2026-02-22T21:53:16.109 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13000 IOs, 6142101504 bytes 2026-02-22T21:53:16.144 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13100 IOs, 6165009408 bytes 2026-02-22T21:53:16.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13200 IOs, 6194229248 bytes 2026-02-22T21:53:16.230 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13300 IOs, 6221421568 bytes 2026-02-22T21:53:16.279 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13400 IOs, 6259118080 bytes 2026-02-22T21:53:16.355 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13500 IOs, 6358465536 bytes 2026-02-22T21:53:16.420 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13600 IOs, 6422323200 bytes 2026-02-22T21:53:16.460 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13700 IOs, 6447728640 bytes 2026-02-22T21:53:16.508 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13800 IOs, 6476843008 bytes 2026-02-22T21:53:16.567 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13900 IOs, 6539424768 bytes 2026-02-22T21:53:16.650 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14000 IOs, 6656066560 bytes 2026-02-22T21:53:16.711 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14100 IOs, 6725734400 bytes 2026-02-22T21:53:16.776 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14200 IOs, 6788970496 bytes 2026-02-22T21:53:16.810 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14300 IOs, 6789592064 bytes 2026-02-22T21:53:16.849 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14400 IOs, 6805288960 bytes 2026-02-22T21:53:16.892 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14500 IOs, 6858417152 bytes 2026-02-22T21:53:16.934 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14600 IOs, 6875485184 bytes 2026-02-22T21:53:17.001 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14700 IOs, 6927577088 bytes 2026-02-22T21:53:17.038 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14800 IOs, 6960034816 bytes 2026-02-22T21:53:17.094 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14900 IOs, 6995155968 bytes 2026-02-22T21:53:17.128 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15000 IOs, 7009378304 bytes 2026-02-22T21:53:17.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15100 IOs, 7043377152 bytes 2026-02-22T21:53:17.229 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15200 IOs, 7085737984 bytes 2026-02-22T21:53:17.301 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15300 IOs, 7162603520 bytes 2026-02-22T21:53:17.366 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15400 IOs, 7233658880 bytes 2026-02-22T21:53:17.426 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15500 IOs, 7326006272 bytes 2026-02-22T21:53:17.487 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15600 IOs, 7372779520 bytes 2026-02-22T21:53:17.549 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15700 IOs, 7426900992 bytes 2026-02-22T21:53:17.625 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15800 IOs, 7508629504 bytes 2026-02-22T21:53:17.681 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15900 IOs, 7581235200 bytes 2026-02-22T21:53:17.750 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16000 IOs, 7656861696 bytes 2026-02-22T21:53:17.802 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16100 IOs, 7699616768 bytes 2026-02-22T21:53:17.890 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16200 IOs, 7791489024 bytes 2026-02-22T21:53:17.941 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16300 IOs, 7835414528 bytes 2026-02-22T21:53:18.004 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16400 IOs, 7895630848 bytes 2026-02-22T21:53:18.048 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16500 IOs, 7930527744 bytes 2026-02-22T21:53:18.102 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16600 IOs, 7996630016 bytes 2026-02-22T21:53:18.163 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ error_code=124 2026-02-22T21:53:18.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set -e 2026-02-22T21:53:18.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' 124 -eq 124 ']' 2026-02-22T21:53:18.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:53:18.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T21:53:18.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_33 2026-02-22T21:53:18.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image_and_enable_mirror cluster2 mirror image_33 snapshot 128M 2026-02-22T21:53:18.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:53:18.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:53:18.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:53:18.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:53:18.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_33 2026-02-22T21:53:18.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:53:18.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T21:53:18.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n snapshot ']' 2026-02-22T21:53:18.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:53:18.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image cluster2 mirror image_33 128M 2026-02-22T21:53:18.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:53:18.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:53:18.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:53:18.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:53:18.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_33 2026-02-22T21:53:18.166 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:53:18.166 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local size=128 2026-02-22T21:53:18.166 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n 128M ']' 2026-02-22T21:53:18.166 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ size=128M 2026-02-22T21:53:18.166 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:53:18.166 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_33 2026-02-22T21:53:18.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T21:53:18.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:53:18.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' image = image ']' 2026-02-22T21:53:18.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:53:18.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ enable_mirror cluster2 mirror image_33 snapshot 2026-02-22T21:53:18.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:53:18.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:53:18.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_33 2026-02-22T21:53:18.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T21:53:18.195 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_33 snapshot 2026-02-22T21:53:18.483 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.2601559589853465) > weight (0.25) 2026-02-22T21:53:18.483 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.9082788521514537) > weight (0.25) 2026-02-22T21:53:18.484 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.35265696572090766) > weight (0.25) 2026-02-22T21:53:18.484 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.3 2026-02-22T21:53:18.484 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Sent signal 15 2026-02-22T21:53:18.484 INFO:tasks.rbd_mirror_thrash:waiting for 6.9335203883606695 secs before reviving daemons 2026-02-22T21:53:18.484 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial026.stderr:2026-02-22T21:53:18.483+0000 7fbff8bae640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.3 (PID: 54449) UID: 1000 2026-02-22T21:53:19.135 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Mirroring enabled 2026-02-22T21:53:19.141 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 info mirror/image_33 2026-02-22T21:53:19.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_33': 2026-02-22T21:53:19.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 128 MiB in 32 objects 2026-02-22T21:53:19.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T21:53:19.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 1 2026-02-22T21:53:19.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 13cc1f20a41d 2026-02-22T21:53:19.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.13cc1f20a41d 2026-02-22T21:53:19.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T21:53:19.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock 2026-02-22T21:53:19.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T21:53:19.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T21:53:19.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:53:18 2026 2026-02-22T21:53:19.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:53:18 2026 2026-02-22T21:53:19.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:53:18 2026 2026-02-22T21:53:19.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T21:53:19.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T21:53:19.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: ad9fe32c-2e40-4486-ab81-178c7a9c3a27 2026-02-22T21:53:19.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: true 2026-02-22T21:53:19.169 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n '' ']' 2026-02-22T21:53:19.170 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ stress_write_image cluster2 mirror image_33 2026-02-22T21:53:19.170 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:53:19.170 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:53:19.170 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_33 2026-02-22T21:53:19.170 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-22T21:53:19.171 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local duration=5 2026-02-22T21:53:19.171 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set +e 2026-02-22T21:53:19.171 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ timeout 5s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_33 --debug-rbd=20 --debug-journaler=20 2026-02-22T21:53:19.243 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:100 IOs, 14690304 bytes 2026-02-22T21:53:19.303 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:200 IOs, 68779008 bytes 2026-02-22T21:53:19.346 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:300 IOs, 92678144 bytes 2026-02-22T21:53:19.436 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:400 IOs, 202152960 bytes 2026-02-22T21:53:19.468 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:500 IOs, 205904896 bytes 2026-02-22T21:53:19.509 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:600 IOs, 228256768 bytes 2026-02-22T21:53:19.570 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:700 IOs, 293401600 bytes 2026-02-22T21:53:19.627 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:800 IOs, 355674112 bytes 2026-02-22T21:53:19.662 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:900 IOs, 369132544 bytes 2026-02-22T21:53:19.725 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1000 IOs, 440668160 bytes 2026-02-22T21:53:19.764 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1100 IOs, 462431232 bytes 2026-02-22T21:53:19.842 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1200 IOs, 548410368 bytes 2026-02-22T21:53:19.901 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1300 IOs, 611088384 bytes 2026-02-22T21:53:19.952 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1400 IOs, 651932672 bytes 2026-02-22T21:53:19.995 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1500 IOs, 680771584 bytes 2026-02-22T21:53:20.054 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1600 IOs, 735096832 bytes 2026-02-22T21:53:20.115 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1700 IOs, 796409856 bytes 2026-02-22T21:53:20.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1800 IOs, 894714880 bytes 2026-02-22T21:53:20.233 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1900 IOs, 920713216 bytes 2026-02-22T21:53:20.314 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2000 IOs, 1018394624 bytes 2026-02-22T21:53:20.362 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2100 IOs, 1061402624 bytes 2026-02-22T21:53:20.424 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2200 IOs, 1137953792 bytes 2026-02-22T21:53:20.492 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2300 IOs, 1216947200 bytes 2026-02-22T21:53:20.560 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2400 IOs, 1274228736 bytes 2026-02-22T21:53:20.621 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2500 IOs, 1354250240 bytes 2026-02-22T21:53:20.712 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2600 IOs, 1433653248 bytes 2026-02-22T21:53:20.776 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2700 IOs, 1510044672 bytes 2026-02-22T21:53:20.825 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2800 IOs, 1555255296 bytes 2026-02-22T21:53:20.907 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2900 IOs, 1622907904 bytes 2026-02-22T21:53:20.977 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3000 IOs, 1673579520 bytes 2026-02-22T21:53:21.018 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3100 IOs, 1711409152 bytes 2026-02-22T21:53:21.106 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3200 IOs, 1807799296 bytes 2026-02-22T21:53:21.150 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3300 IOs, 1835673600 bytes 2026-02-22T21:53:21.213 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3400 IOs, 1912404992 bytes 2026-02-22T21:53:21.258 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3500 IOs, 1913026560 bytes 2026-02-22T21:53:21.291 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3600 IOs, 1928061952 bytes 2026-02-22T21:53:21.356 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3700 IOs, 2002384896 bytes 2026-02-22T21:53:21.400 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3800 IOs, 2023445504 bytes 2026-02-22T21:53:21.437 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3900 IOs, 2032535552 bytes 2026-02-22T21:53:21.473 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4000 IOs, 2042258432 bytes 2026-02-22T21:53:21.537 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4100 IOs, 2106470400 bytes 2026-02-22T21:53:21.611 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4200 IOs, 2168672256 bytes 2026-02-22T21:53:21.673 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4300 IOs, 2250129408 bytes 2026-02-22T21:53:21.735 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4400 IOs, 2291430400 bytes 2026-02-22T21:53:21.780 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4500 IOs, 2327374848 bytes 2026-02-22T21:53:21.833 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4600 IOs, 2373832704 bytes 2026-02-22T21:53:21.896 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4700 IOs, 2416565248 bytes 2026-02-22T21:53:21.968 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4800 IOs, 2497350656 bytes 2026-02-22T21:53:22.028 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4900 IOs, 2551720960 bytes 2026-02-22T21:53:22.089 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5000 IOs, 2606524416 bytes 2026-02-22T21:53:22.147 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5100 IOs, 2670826496 bytes 2026-02-22T21:53:22.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5200 IOs, 2686666752 bytes 2026-02-22T21:53:22.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5300 IOs, 2770951168 bytes 2026-02-22T21:53:22.291 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5400 IOs, 2792480768 bytes 2026-02-22T21:53:22.338 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5500 IOs, 2833789952 bytes 2026-02-22T21:53:22.379 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5600 IOs, 2855965696 bytes 2026-02-22T21:53:22.409 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5700 IOs, 2872006656 bytes 2026-02-22T21:53:22.451 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5800 IOs, 2893948928 bytes 2026-02-22T21:53:22.493 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5900 IOs, 2931625984 bytes 2026-02-22T21:53:22.536 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6000 IOs, 2945161216 bytes 2026-02-22T21:53:22.619 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6100 IOs, 3040066560 bytes 2026-02-22T21:53:22.686 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6200 IOs, 3127650304 bytes 2026-02-22T21:53:22.777 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6300 IOs, 3239061504 bytes 2026-02-22T21:53:22.837 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6400 IOs, 3289358336 bytes 2026-02-22T21:53:22.888 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6500 IOs, 3311548416 bytes 2026-02-22T21:53:22.930 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6600 IOs, 3330395136 bytes 2026-02-22T21:53:22.970 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6700 IOs, 3354304512 bytes 2026-02-22T21:53:23.013 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6800 IOs, 3377031168 bytes 2026-02-22T21:53:23.085 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6900 IOs, 3452732416 bytes 2026-02-22T21:53:23.133 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7000 IOs, 3493576704 bytes 2026-02-22T21:53:23.184 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7100 IOs, 3521400832 bytes 2026-02-22T21:53:23.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7200 IOs, 3619726336 bytes 2026-02-22T21:53:23.312 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7300 IOs, 3648138240 bytes 2026-02-22T21:53:23.372 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7400 IOs, 3715645440 bytes 2026-02-22T21:53:23.407 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7500 IOs, 3727308800 bytes 2026-02-22T21:53:23.435 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7600 IOs, 3727913984 bytes 2026-02-22T21:53:23.518 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7700 IOs, 3833207808 bytes 2026-02-22T21:53:23.562 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7800 IOs, 3859641344 bytes 2026-02-22T21:53:23.606 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7900 IOs, 3906835456 bytes 2026-02-22T21:53:23.657 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8000 IOs, 3931161600 bytes 2026-02-22T21:53:23.707 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8100 IOs, 3977297920 bytes 2026-02-22T21:53:23.787 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.3 2026-02-22T21:53:23.787 INFO:teuthology.orchestra.run:waiting for 600 2026-02-22T21:53:23.788 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.3 2026-02-22T21:53:23.788 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Restarting daemon 2026-02-22T21:53:23.788 DEBUG:teuthology.orchestra.run.trial026:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.3 2026-02-22T21:53:23.789 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8200 IOs, 4066241536 bytes 2026-02-22T21:53:23.791 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Started 2026-02-22T21:53:23.792 INFO:tasks.rbd_mirror_thrash:waiting for 60.37692268363005 secs before thrashing 2026-02-22T21:53:23.834 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8300 IOs, 4099524608 bytes 2026-02-22T21:53:23.886 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8400 IOs, 4117405696 bytes 2026-02-22T21:53:23.952 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8500 IOs, 4176472064 bytes 2026-02-22T21:53:24.000 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8600 IOs, 4198796288 bytes 2026-02-22T21:53:24.035 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8700 IOs, 4221362176 bytes 2026-02-22T21:53:24.111 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8800 IOs, 4307688448 bytes 2026-02-22T21:53:24.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ error_code=124 2026-02-22T21:53:24.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set -e 2026-02-22T21:53:24.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' 124 -eq 124 ']' 2026-02-22T21:53:24.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:53:24.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T21:53:24.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_34 2026-02-22T21:53:24.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image_and_enable_mirror cluster2 mirror image_34 snapshot 128M 2026-02-22T21:53:24.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:53:24.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:53:24.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:53:24.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:53:24.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_34 2026-02-22T21:53:24.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:53:24.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T21:53:24.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n snapshot ']' 2026-02-22T21:53:24.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:53:24.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image cluster2 mirror image_34 128M 2026-02-22T21:53:24.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:53:24.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:53:24.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:53:24.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:53:24.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_34 2026-02-22T21:53:24.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:53:24.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local size=128 2026-02-22T21:53:24.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n 128M ']' 2026-02-22T21:53:24.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ size=128M 2026-02-22T21:53:24.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:53:24.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_34 2026-02-22T21:53:24.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T21:53:24.209 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:53:24.209 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' image = image ']' 2026-02-22T21:53:24.209 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:53:24.209 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ enable_mirror cluster2 mirror image_34 snapshot 2026-02-22T21:53:24.209 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:53:24.209 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:53:24.209 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_34 2026-02-22T21:53:24.209 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T21:53:24.209 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_34 snapshot 2026-02-22T21:53:25.134 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Mirroring enabled 2026-02-22T21:53:25.139 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 info mirror/image_34 2026-02-22T21:53:25.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_34': 2026-02-22T21:53:25.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 128 MiB in 32 objects 2026-02-22T21:53:25.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T21:53:25.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 1 2026-02-22T21:53:25.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 13dfc9d697f7 2026-02-22T21:53:25.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.13dfc9d697f7 2026-02-22T21:53:25.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T21:53:25.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock 2026-02-22T21:53:25.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T21:53:25.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T21:53:25.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:53:24 2026 2026-02-22T21:53:25.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:53:24 2026 2026-02-22T21:53:25.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:53:24 2026 2026-02-22T21:53:25.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T21:53:25.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T21:53:25.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: 787c83b7-1192-4f94-aa31-23be72fbd343 2026-02-22T21:53:25.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: true 2026-02-22T21:53:25.166 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n '' ']' 2026-02-22T21:53:25.166 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ stress_write_image cluster2 mirror image_34 2026-02-22T21:53:25.166 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:53:25.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:53:25.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_34 2026-02-22T21:53:25.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-22T21:53:25.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local duration=9 2026-02-22T21:53:25.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set +e 2026-02-22T21:53:25.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ timeout 9s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_34 --debug-rbd=20 --debug-journaler=20 2026-02-22T21:53:25.249 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:100 IOs, 29782016 bytes 2026-02-22T21:53:25.282 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:200 IOs, 55958528 bytes 2026-02-22T21:53:25.317 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:300 IOs, 56591360 bytes 2026-02-22T21:53:25.365 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:400 IOs, 100049920 bytes 2026-02-22T21:53:25.418 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.3 2026-02-22T21:53:25.418 INFO:teuthology.orchestra.run:waiting for 600 2026-02-22T21:53:25.418 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.3 2026-02-22T21:53:25.418 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Restarting daemon 2026-02-22T21:53:25.418 DEBUG:teuthology.orchestra.run.trial026:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.3 2026-02-22T21:53:25.419 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:500 IOs, 146352128 bytes 2026-02-22T21:53:25.420 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Started 2026-02-22T21:53:25.420 INFO:tasks.rbd_mirror_thrash:waiting for 78.55249556958015 secs before thrashing 2026-02-22T21:53:25.468 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:600 IOs, 189403136 bytes 2026-02-22T21:53:25.544 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:700 IOs, 280070144 bytes 2026-02-22T21:53:25.615 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:800 IOs, 345306112 bytes 2026-02-22T21:53:25.673 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:900 IOs, 399597568 bytes 2026-02-22T21:53:25.702 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1000 IOs, 400238592 bytes 2026-02-22T21:53:25.748 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1100 IOs, 439025664 bytes 2026-02-22T21:53:25.803 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1200 IOs, 480553984 bytes 2026-02-22T21:53:25.886 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1300 IOs, 586183680 bytes 2026-02-22T21:53:25.963 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1400 IOs, 667865088 bytes 2026-02-22T21:53:26.013 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1500 IOs, 721463296 bytes 2026-02-22T21:53:26.053 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1600 IOs, 739043328 bytes 2026-02-22T21:53:26.086 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1700 IOs, 744765440 bytes 2026-02-22T21:53:26.138 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1800 IOs, 793394176 bytes 2026-02-22T21:53:26.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1900 IOs, 855436288 bytes 2026-02-22T21:53:26.254 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2000 IOs, 924820480 bytes 2026-02-22T21:53:26.298 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2100 IOs, 971317248 bytes 2026-02-22T21:53:26.334 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2200 IOs, 972232704 bytes 2026-02-22T21:53:26.371 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2300 IOs, 996986880 bytes 2026-02-22T21:53:26.434 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2400 IOs, 1063774208 bytes 2026-02-22T21:53:26.463 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2500 IOs, 1068294144 bytes 2026-02-22T21:53:26.501 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2600 IOs, 1089420288 bytes 2026-02-22T21:53:26.559 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2700 IOs, 1138884608 bytes 2026-02-22T21:53:26.603 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2800 IOs, 1161476096 bytes 2026-02-22T21:53:26.657 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2900 IOs, 1208261632 bytes 2026-02-22T21:53:26.707 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3000 IOs, 1244758016 bytes 2026-02-22T21:53:26.765 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3100 IOs, 1313486848 bytes 2026-02-22T21:53:26.802 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3200 IOs, 1326929920 bytes 2026-02-22T21:53:26.857 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3300 IOs, 1372831744 bytes 2026-02-22T21:53:26.912 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3400 IOs, 1409508352 bytes 2026-02-22T21:53:26.956 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3500 IOs, 1441456128 bytes 2026-02-22T21:53:26.996 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3600 IOs, 1460383744 bytes 2026-02-22T21:53:27.049 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3700 IOs, 1504964608 bytes 2026-02-22T21:53:27.090 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3800 IOs, 1526279168 bytes 2026-02-22T21:53:27.141 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3900 IOs, 1576092672 bytes 2026-02-22T21:53:27.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4000 IOs, 1635856384 bytes 2026-02-22T21:53:27.230 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4100 IOs, 1636473856 bytes 2026-02-22T21:53:27.258 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4200 IOs, 1637042176 bytes 2026-02-22T21:53:27.299 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4300 IOs, 1668107264 bytes 2026-02-22T21:53:27.335 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4400 IOs, 1692295168 bytes 2026-02-22T21:53:27.390 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4500 IOs, 1728214016 bytes 2026-02-22T21:53:27.430 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4600 IOs, 1754314752 bytes 2026-02-22T21:53:27.467 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4700 IOs, 1770156032 bytes 2026-02-22T21:53:27.505 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4800 IOs, 1802387456 bytes 2026-02-22T21:53:27.582 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4900 IOs, 1875519488 bytes 2026-02-22T21:53:27.649 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5000 IOs, 1931691008 bytes 2026-02-22T21:53:27.719 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5100 IOs, 2022078464 bytes 2026-02-22T21:53:27.777 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5200 IOs, 2071179264 bytes 2026-02-22T21:53:27.853 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5300 IOs, 2153859072 bytes 2026-02-22T21:53:27.888 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5400 IOs, 2164338688 bytes 2026-02-22T21:53:27.950 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5500 IOs, 2204798976 bytes 2026-02-22T21:53:28.012 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5600 IOs, 2265378816 bytes 2026-02-22T21:53:28.059 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5700 IOs, 2300210176 bytes 2026-02-22T21:53:28.107 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5800 IOs, 2327976960 bytes 2026-02-22T21:53:28.159 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5900 IOs, 2376463360 bytes 2026-02-22T21:53:28.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6000 IOs, 2416995328 bytes 2026-02-22T21:53:28.243 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6100 IOs, 2419318784 bytes 2026-02-22T21:53:28.294 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6200 IOs, 2464125952 bytes 2026-02-22T21:53:28.327 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6300 IOs, 2473181184 bytes 2026-02-22T21:53:28.367 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6400 IOs, 2486431744 bytes 2026-02-22T21:53:28.433 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6500 IOs, 2555330560 bytes 2026-02-22T21:53:28.481 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6600 IOs, 2607085568 bytes 2026-02-22T21:53:28.522 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6700 IOs, 2622088192 bytes 2026-02-22T21:53:28.573 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6800 IOs, 2659826688 bytes 2026-02-22T21:53:28.630 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6900 IOs, 2698134528 bytes 2026-02-22T21:53:28.680 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7000 IOs, 2727460864 bytes 2026-02-22T21:53:28.736 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7100 IOs, 2786100224 bytes 2026-02-22T21:53:28.833 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7200 IOs, 2918497280 bytes 2026-02-22T21:53:28.872 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7300 IOs, 2936846336 bytes 2026-02-22T21:53:28.962 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7400 IOs, 3019693056 bytes 2026-02-22T21:53:29.024 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7500 IOs, 3081818112 bytes 2026-02-22T21:53:29.064 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7600 IOs, 3095715840 bytes 2026-02-22T21:53:29.109 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7700 IOs, 3120776192 bytes 2026-02-22T21:53:29.145 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7800 IOs, 3133584384 bytes 2026-02-22T21:53:29.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7900 IOs, 3180653568 bytes 2026-02-22T21:53:29.233 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8000 IOs, 3184736256 bytes 2026-02-22T21:53:29.311 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8100 IOs, 3292819456 bytes 2026-02-22T21:53:29.355 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8200 IOs, 3316902912 bytes 2026-02-22T21:53:29.408 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8300 IOs, 3358162944 bytes 2026-02-22T21:53:29.458 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8400 IOs, 3392800768 bytes 2026-02-22T21:53:29.537 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8500 IOs, 3501780992 bytes 2026-02-22T21:53:29.597 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8600 IOs, 3550588928 bytes 2026-02-22T21:53:29.649 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8700 IOs, 3587489792 bytes 2026-02-22T21:53:29.698 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8800 IOs, 3617436672 bytes 2026-02-22T21:53:29.738 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8900 IOs, 3638358016 bytes 2026-02-22T21:53:29.769 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9000 IOs, 3638974464 bytes 2026-02-22T21:53:29.800 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9100 IOs, 3639592960 bytes 2026-02-22T21:53:29.858 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9200 IOs, 3703546880 bytes 2026-02-22T21:53:29.898 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9300 IOs, 3734583296 bytes 2026-02-22T21:53:29.964 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9400 IOs, 3763854336 bytes 2026-02-22T21:53:30.010 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9500 IOs, 3787870208 bytes 2026-02-22T21:53:30.060 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9600 IOs, 3830808576 bytes 2026-02-22T21:53:30.110 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9700 IOs, 3863590912 bytes 2026-02-22T21:53:30.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9800 IOs, 3946213376 bytes 2026-02-22T21:53:30.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9900 IOs, 3946800128 bytes 2026-02-22T21:53:30.244 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10000 IOs, 3952438272 bytes 2026-02-22T21:53:30.275 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10100 IOs, 3953058816 bytes 2026-02-22T21:53:30.312 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10200 IOs, 3966523392 bytes 2026-02-22T21:53:30.354 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10300 IOs, 3992518656 bytes 2026-02-22T21:53:30.396 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10400 IOs, 4006203392 bytes 2026-02-22T21:53:30.450 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10500 IOs, 4059917312 bytes 2026-02-22T21:53:30.489 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10600 IOs, 4069019648 bytes 2026-02-22T21:53:30.535 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10700 IOs, 4103114752 bytes 2026-02-22T21:53:30.588 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10800 IOs, 4154995712 bytes 2026-02-22T21:53:30.663 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10900 IOs, 4223243264 bytes 2026-02-22T21:53:30.712 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11000 IOs, 4258344960 bytes 2026-02-22T21:53:30.758 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11100 IOs, 4288388096 bytes 2026-02-22T21:53:30.847 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11200 IOs, 4394740736 bytes 2026-02-22T21:53:30.891 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11300 IOs, 4417396736 bytes 2026-02-22T21:53:30.938 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11400 IOs, 4450072576 bytes 2026-02-22T21:53:30.981 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11500 IOs, 4474395648 bytes 2026-02-22T21:53:31.025 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11600 IOs, 4484761600 bytes 2026-02-22T21:53:31.064 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11700 IOs, 4509906944 bytes 2026-02-22T21:53:31.118 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11800 IOs, 4551012352 bytes 2026-02-22T21:53:31.160 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11900 IOs, 4566437888 bytes 2026-02-22T21:53:31.213 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12000 IOs, 4633896960 bytes 2026-02-22T21:53:31.283 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12100 IOs, 4678461440 bytes 2026-02-22T21:53:31.387 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12200 IOs, 4834463744 bytes 2026-02-22T21:53:31.460 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12300 IOs, 4930394112 bytes 2026-02-22T21:53:31.504 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12400 IOs, 4956780544 bytes 2026-02-22T21:53:31.565 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12500 IOs, 5023817728 bytes 2026-02-22T21:53:31.601 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12600 IOs, 5037070336 bytes 2026-02-22T21:53:31.656 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12700 IOs, 5070700544 bytes 2026-02-22T21:53:31.705 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12800 IOs, 5100793856 bytes 2026-02-22T21:53:31.753 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12900 IOs, 5144639488 bytes 2026-02-22T21:53:31.818 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13000 IOs, 5226577920 bytes 2026-02-22T21:53:31.884 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13100 IOs, 5277433856 bytes 2026-02-22T21:53:31.952 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13200 IOs, 5343486976 bytes 2026-02-22T21:53:32.028 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13300 IOs, 5430571008 bytes 2026-02-22T21:53:32.082 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13400 IOs, 5460477952 bytes 2026-02-22T21:53:32.121 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13500 IOs, 5475115008 bytes 2026-02-22T21:53:32.161 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13600 IOs, 5494971392 bytes 2026-02-22T21:53:32.218 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13700 IOs, 5547536384 bytes 2026-02-22T21:53:32.280 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13800 IOs, 5607533568 bytes 2026-02-22T21:53:32.325 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13900 IOs, 5635353600 bytes 2026-02-22T21:53:32.367 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14000 IOs, 5661135872 bytes 2026-02-22T21:53:32.415 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14100 IOs, 5694425088 bytes 2026-02-22T21:53:32.469 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14200 IOs, 5772155904 bytes 2026-02-22T21:53:32.518 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14300 IOs, 5782740992 bytes 2026-02-22T21:53:32.579 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14400 IOs, 5835336704 bytes 2026-02-22T21:53:32.623 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14500 IOs, 5859492864 bytes 2026-02-22T21:53:32.667 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14600 IOs, 5880027136 bytes 2026-02-22T21:53:32.712 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14700 IOs, 5904927744 bytes 2026-02-22T21:53:32.773 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14800 IOs, 5960044544 bytes 2026-02-22T21:53:32.834 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14900 IOs, 6014498816 bytes 2026-02-22T21:53:32.938 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15000 IOs, 6172846080 bytes 2026-02-22T21:53:33.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15100 IOs, 6208498688 bytes 2026-02-22T21:53:33.231 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15200 IOs, 6209111040 bytes 2026-02-22T21:53:33.277 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15300 IOs, 6240523264 bytes 2026-02-22T21:53:33.335 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15400 IOs, 6282822656 bytes 2026-02-22T21:53:33.374 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15500 IOs, 6301075456 bytes 2026-02-22T21:53:33.430 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15600 IOs, 6365412352 bytes 2026-02-22T21:53:33.479 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15700 IOs, 6383890432 bytes 2026-02-22T21:53:33.529 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15800 IOs, 6429252608 bytes 2026-02-22T21:53:33.531 INFO:tasks.ceph.cluster2.osd.0.trial026.stderr:2026-02-22T21:53:33.528+0000 7ff4b543f640 -1 reset not still connected to 0x55b3803dedd0 2026-02-22T21:53:33.600 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15900 IOs, 6523811840 bytes 2026-02-22T21:53:33.662 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16000 IOs, 6564415488 bytes 2026-02-22T21:53:33.713 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16100 IOs, 6613120000 bytes 2026-02-22T21:53:33.772 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16200 IOs, 6658124800 bytes 2026-02-22T21:53:33.812 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16300 IOs, 6679892992 bytes 2026-02-22T21:53:33.865 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16400 IOs, 6725470208 bytes 2026-02-22T21:53:33.901 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16500 IOs, 6741376000 bytes 2026-02-22T21:53:33.941 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16600 IOs, 6770585600 bytes 2026-02-22T21:53:33.997 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16700 IOs, 6810315776 bytes 2026-02-22T21:53:34.055 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16800 IOs, 6842176512 bytes 2026-02-22T21:53:34.094 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16900 IOs, 6859723776 bytes 2026-02-22T21:53:34.132 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17000 IOs, 6869971968 bytes 2026-02-22T21:53:34.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ error_code=124 2026-02-22T21:53:34.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set -e 2026-02-22T21:53:34.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' 124 -eq 124 ']' 2026-02-22T21:53:34.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:53:34.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T21:53:34.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_35 2026-02-22T21:53:34.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image_and_enable_mirror cluster2 mirror image_35 snapshot 128M 2026-02-22T21:53:34.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:53:34.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:53:34.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:53:34.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:53:34.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_35 2026-02-22T21:53:34.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:53:34.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T21:53:34.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n snapshot ']' 2026-02-22T21:53:34.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:53:34.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image cluster2 mirror image_35 128M 2026-02-22T21:53:34.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:53:34.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:53:34.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:53:34.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:53:34.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_35 2026-02-22T21:53:34.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:53:34.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local size=128 2026-02-22T21:53:34.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n 128M ']' 2026-02-22T21:53:34.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ size=128M 2026-02-22T21:53:34.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:53:34.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_35 2026-02-22T21:53:34.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T21:53:34.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:53:34.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' image = image ']' 2026-02-22T21:53:34.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:53:34.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ enable_mirror cluster2 mirror image_35 snapshot 2026-02-22T21:53:34.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:53:34.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:53:34.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_35 2026-02-22T21:53:34.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T21:53:34.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_35 snapshot 2026-02-22T21:53:35.145 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Mirroring enabled 2026-02-22T21:53:35.151 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 info mirror/image_35 2026-02-22T21:53:35.175 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_35': 2026-02-22T21:53:35.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 128 MiB in 32 objects 2026-02-22T21:53:35.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T21:53:35.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 1 2026-02-22T21:53:35.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 13ef40f026f8 2026-02-22T21:53:35.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.13ef40f026f8 2026-02-22T21:53:35.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T21:53:35.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock 2026-02-22T21:53:35.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T21:53:35.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T21:53:35.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:53:34 2026 2026-02-22T21:53:35.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:53:34 2026 2026-02-22T21:53:35.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:53:34 2026 2026-02-22T21:53:35.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T21:53:35.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T21:53:35.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: 7d8cf4a3-1fa1-4dcc-9d4d-2c1a44d0099a 2026-02-22T21:53:35.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: true 2026-02-22T21:53:35.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n '' ']' 2026-02-22T21:53:35.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ stress_write_image cluster2 mirror image_35 2026-02-22T21:53:35.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:53:35.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:53:35.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_35 2026-02-22T21:53:35.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-22T21:53:35.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local duration=7 2026-02-22T21:53:35.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set +e 2026-02-22T21:53:35.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ timeout 7s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_35 --debug-rbd=20 --debug-journaler=20 2026-02-22T21:53:35.256 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:100 IOs, 23880704 bytes 2026-02-22T21:53:35.305 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:200 IOs, 62862336 bytes 2026-02-22T21:53:35.343 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:300 IOs, 81985536 bytes 2026-02-22T21:53:35.382 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:400 IOs, 100995072 bytes 2026-02-22T21:53:35.436 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:500 IOs, 165881856 bytes 2026-02-22T21:53:35.506 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:600 IOs, 242883584 bytes 2026-02-22T21:53:35.545 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:700 IOs, 258862080 bytes 2026-02-22T21:53:35.587 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:800 IOs, 283529216 bytes 2026-02-22T21:53:35.624 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:900 IOs, 297164800 bytes 2026-02-22T21:53:35.680 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1000 IOs, 350653440 bytes 2026-02-22T21:53:35.722 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1100 IOs, 374695936 bytes 2026-02-22T21:53:35.763 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1200 IOs, 400306176 bytes 2026-02-22T21:53:35.800 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1300 IOs, 418970624 bytes 2026-02-22T21:53:35.857 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1400 IOs, 482094080 bytes 2026-02-22T21:53:35.910 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1500 IOs, 535317504 bytes 2026-02-22T21:53:35.949 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1600 IOs, 577044480 bytes 2026-02-22T21:53:36.008 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1700 IOs, 603834368 bytes 2026-02-22T21:53:36.095 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1800 IOs, 720302080 bytes 2026-02-22T21:53:36.125 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1900 IOs, 724590592 bytes 2026-02-22T21:53:36.186 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2000 IOs, 787395584 bytes 2026-02-22T21:53:36.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2100 IOs, 804071424 bytes 2026-02-22T21:53:36.264 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2200 IOs, 829449216 bytes 2026-02-22T21:53:36.322 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2300 IOs, 893692928 bytes 2026-02-22T21:53:36.350 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2400 IOs, 915172352 bytes 2026-02-22T21:53:36.425 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2500 IOs, 989418496 bytes 2026-02-22T21:53:36.467 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2600 IOs, 1019827200 bytes 2026-02-22T21:53:36.497 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2700 IOs, 1020438528 bytes 2026-02-22T21:53:36.556 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2800 IOs, 1072371712 bytes 2026-02-22T21:53:36.595 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2900 IOs, 1117707264 bytes 2026-02-22T21:53:36.663 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3000 IOs, 1171395584 bytes 2026-02-22T21:53:36.726 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3100 IOs, 1245934592 bytes 2026-02-22T21:53:36.779 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3200 IOs, 1281047552 bytes 2026-02-22T21:53:36.824 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3300 IOs, 1315427328 bytes 2026-02-22T21:53:36.872 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3400 IOs, 1361746944 bytes 2026-02-22T21:53:36.923 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3500 IOs, 1397534720 bytes 2026-02-22T21:53:36.963 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3600 IOs, 1427255296 bytes 2026-02-22T21:53:37.019 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3700 IOs, 1462312960 bytes 2026-02-22T21:53:37.064 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3800 IOs, 1501302784 bytes 2026-02-22T21:53:37.102 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3900 IOs, 1521522688 bytes 2026-02-22T21:53:37.153 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4000 IOs, 1567128576 bytes 2026-02-22T21:53:37.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4100 IOs, 1641368576 bytes 2026-02-22T21:53:37.286 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4200 IOs, 1718185984 bytes 2026-02-22T21:53:37.317 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4300 IOs, 1718814720 bytes 2026-02-22T21:53:37.351 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4400 IOs, 1745774592 bytes 2026-02-22T21:53:37.413 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4500 IOs, 1807668224 bytes 2026-02-22T21:53:37.471 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4600 IOs, 1869529088 bytes 2026-02-22T21:53:37.530 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4700 IOs, 1923045376 bytes 2026-02-22T21:53:37.593 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4800 IOs, 2002572288 bytes 2026-02-22T21:53:37.625 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4900 IOs, 2011826176 bytes 2026-02-22T21:53:37.676 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5000 IOs, 2053255168 bytes 2026-02-22T21:53:37.729 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5100 IOs, 2090330112 bytes 2026-02-22T21:53:37.799 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5200 IOs, 2168695808 bytes 2026-02-22T21:53:37.857 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5300 IOs, 2219327488 bytes 2026-02-22T21:53:37.900 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5400 IOs, 2261338112 bytes 2026-02-22T21:53:37.949 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5500 IOs, 2297266176 bytes 2026-02-22T21:53:37.997 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5600 IOs, 2320736256 bytes 2026-02-22T21:53:38.051 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5700 IOs, 2338214912 bytes 2026-02-22T21:53:38.122 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5800 IOs, 2431370240 bytes 2026-02-22T21:53:38.185 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5900 IOs, 2472172544 bytes 2026-02-22T21:53:38.256 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6000 IOs, 2561208320 bytes 2026-02-22T21:53:38.342 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6100 IOs, 2663150592 bytes 2026-02-22T21:53:38.380 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6200 IOs, 2679397376 bytes 2026-02-22T21:53:38.415 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6300 IOs, 2692670464 bytes 2026-02-22T21:53:38.427 INFO:tasks.ceph.cluster2.osd.2.trial026.stderr:2026-02-22T21:53:38.424+0000 7f32e29d2640 -1 reset not still connected to 0x5576aad62b60 2026-02-22T21:53:38.427 INFO:tasks.ceph.cluster2.osd.2.trial026.stderr:2026-02-22T21:53:38.424+0000 7f32e29d2640 -1 reset not still connected to 0x5576abccba00 2026-02-22T21:53:38.450 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6400 IOs, 2704168960 bytes 2026-02-22T21:53:38.520 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6500 IOs, 2796181504 bytes 2026-02-22T21:53:38.582 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6600 IOs, 2849860608 bytes 2026-02-22T21:53:38.623 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6700 IOs, 2895919104 bytes 2026-02-22T21:53:38.691 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6800 IOs, 2953032704 bytes 2026-02-22T21:53:38.725 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6900 IOs, 2963582976 bytes 2026-02-22T21:53:38.772 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7000 IOs, 3003024384 bytes 2026-02-22T21:53:38.823 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7100 IOs, 3045080064 bytes 2026-02-22T21:53:38.884 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7200 IOs, 3112103936 bytes 2026-02-22T21:53:38.944 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7300 IOs, 3175069696 bytes 2026-02-22T21:53:38.989 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7400 IOs, 3206789120 bytes 2026-02-22T21:53:39.051 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7500 IOs, 3264818176 bytes 2026-02-22T21:53:39.095 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7600 IOs, 3286417408 bytes 2026-02-22T21:53:39.130 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7700 IOs, 3304377344 bytes 2026-02-22T21:53:39.186 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7800 IOs, 3354383360 bytes 2026-02-22T21:53:39.261 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7900 IOs, 3437963264 bytes 2026-02-22T21:53:39.302 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8000 IOs, 3467142144 bytes 2026-02-22T21:53:39.357 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8100 IOs, 3513206784 bytes 2026-02-22T21:53:39.405 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8200 IOs, 3544397824 bytes 2026-02-22T21:53:39.452 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8300 IOs, 3578341376 bytes 2026-02-22T21:53:39.483 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8400 IOs, 3590776832 bytes 2026-02-22T21:53:39.547 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8500 IOs, 3660578816 bytes 2026-02-22T21:53:39.604 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8600 IOs, 3698014208 bytes 2026-02-22T21:53:39.640 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8700 IOs, 3727193088 bytes 2026-02-22T21:53:39.713 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8800 IOs, 3806651392 bytes 2026-02-22T21:53:39.784 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8900 IOs, 3894249472 bytes 2026-02-22T21:53:39.873 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9000 IOs, 4000508928 bytes 2026-02-22T21:53:39.918 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9100 IOs, 4024615936 bytes 2026-02-22T21:53:39.960 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9200 IOs, 4059167744 bytes 2026-02-22T21:53:40.020 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9300 IOs, 4095892480 bytes 2026-02-22T21:53:40.078 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9400 IOs, 4146715648 bytes 2026-02-22T21:53:40.123 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9500 IOs, 4164271104 bytes 2026-02-22T21:53:40.185 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9600 IOs, 4226971648 bytes 2026-02-22T21:53:40.223 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9700 IOs, 4243153920 bytes 2026-02-22T21:53:40.250 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9800 IOs, 4243782656 bytes 2026-02-22T21:53:40.310 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9900 IOs, 4300055552 bytes 2026-02-22T21:53:40.349 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10000 IOs, 4323319808 bytes 2026-02-22T21:53:40.399 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10100 IOs, 4372209664 bytes 2026-02-22T21:53:40.450 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10200 IOs, 4414489600 bytes 2026-02-22T21:53:40.498 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10300 IOs, 4474179584 bytes 2026-02-22T21:53:40.552 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10400 IOs, 4502701056 bytes 2026-02-22T21:53:40.605 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10500 IOs, 4553431040 bytes 2026-02-22T21:53:40.660 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10600 IOs, 4594193408 bytes 2026-02-22T21:53:40.715 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10700 IOs, 4647144448 bytes 2026-02-22T21:53:40.758 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10800 IOs, 4668127232 bytes 2026-02-22T21:53:40.803 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10900 IOs, 4693855232 bytes 2026-02-22T21:53:40.859 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11000 IOs, 4753092608 bytes 2026-02-22T21:53:40.953 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11100 IOs, 4886284288 bytes 2026-02-22T21:53:40.998 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11200 IOs, 4913827840 bytes 2026-02-22T21:53:41.036 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11300 IOs, 4930704384 bytes 2026-02-22T21:53:41.113 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11400 IOs, 5012676608 bytes 2026-02-22T21:53:41.171 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11500 IOs, 5070523392 bytes 2026-02-22T21:53:41.250 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11600 IOs, 5159067648 bytes 2026-02-22T21:53:41.312 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11700 IOs, 5224209408 bytes 2026-02-22T21:53:41.362 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11800 IOs, 5264617472 bytes 2026-02-22T21:53:41.417 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11900 IOs, 5318139904 bytes 2026-02-22T21:53:41.444 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12000 IOs, 5318722560 bytes 2026-02-22T21:53:41.490 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12100 IOs, 5355561984 bytes 2026-02-22T21:53:41.543 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12200 IOs, 5400519680 bytes 2026-02-22T21:53:41.613 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12300 IOs, 5485648896 bytes 2026-02-22T21:53:41.688 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12400 IOs, 5579530240 bytes 2026-02-22T21:53:41.741 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12500 IOs, 5618869248 bytes 2026-02-22T21:53:41.790 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12600 IOs, 5655929856 bytes 2026-02-22T21:53:41.819 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12700 IOs, 5660266496 bytes 2026-02-22T21:53:41.868 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12800 IOs, 5702298624 bytes 2026-02-22T21:53:41.909 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12900 IOs, 5723320320 bytes 2026-02-22T21:53:41.953 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13000 IOs, 5755165696 bytes 2026-02-22T21:53:41.987 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13100 IOs, 5765740544 bytes 2026-02-22T21:53:42.031 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13200 IOs, 5791108096 bytes 2026-02-22T21:53:42.062 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13300 IOs, 5801758720 bytes 2026-02-22T21:53:42.104 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13400 IOs, 5809305600 bytes 2026-02-22T21:53:42.154 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13500 IOs, 5851242496 bytes 2026-02-22T21:53:42.185 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ error_code=124 2026-02-22T21:53:42.185 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set -e 2026-02-22T21:53:42.185 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' 124 -eq 124 ']' 2026-02-22T21:53:42.185 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:53:42.185 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T21:53:42.185 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_36 2026-02-22T21:53:42.186 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image_and_enable_mirror cluster2 mirror image_36 snapshot 128M 2026-02-22T21:53:42.186 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:53:42.186 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:53:42.186 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:53:42.186 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:53:42.186 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_36 2026-02-22T21:53:42.186 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:53:42.186 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T21:53:42.186 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n snapshot ']' 2026-02-22T21:53:42.186 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:53:42.186 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image cluster2 mirror image_36 128M 2026-02-22T21:53:42.186 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:53:42.187 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:53:42.187 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:53:42.187 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:53:42.187 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_36 2026-02-22T21:53:42.187 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:53:42.187 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local size=128 2026-02-22T21:53:42.187 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n 128M ']' 2026-02-22T21:53:42.187 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ size=128M 2026-02-22T21:53:42.187 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:53:42.187 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_36 2026-02-22T21:53:42.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T21:53:42.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:53:42.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' image = image ']' 2026-02-22T21:53:42.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:53:42.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ enable_mirror cluster2 mirror image_36 snapshot 2026-02-22T21:53:42.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:53:42.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:53:42.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_36 2026-02-22T21:53:42.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T21:53:42.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_36 snapshot 2026-02-22T21:53:43.162 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Mirroring enabled 2026-02-22T21:53:43.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 info mirror/image_36 2026-02-22T21:53:43.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_36': 2026-02-22T21:53:43.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 128 MiB in 32 objects 2026-02-22T21:53:43.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T21:53:43.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 1 2026-02-22T21:53:43.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 13fb35d23984 2026-02-22T21:53:43.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.13fb35d23984 2026-02-22T21:53:43.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T21:53:43.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock 2026-02-22T21:53:43.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T21:53:43.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T21:53:43.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:53:42 2026 2026-02-22T21:53:43.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:53:42 2026 2026-02-22T21:53:43.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:53:42 2026 2026-02-22T21:53:43.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T21:53:43.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T21:53:43.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: 9860c66a-05af-466f-baca-3772f5e211ad 2026-02-22T21:53:43.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: true 2026-02-22T21:53:43.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n '' ']' 2026-02-22T21:53:43.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ stress_write_image cluster2 mirror image_36 2026-02-22T21:53:43.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:53:43.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:53:43.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_36 2026-02-22T21:53:43.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-22T21:53:43.195 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local duration=11 2026-02-22T21:53:43.195 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set +e 2026-02-22T21:53:43.195 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ timeout 11s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_36 --debug-rbd=20 --debug-journaler=20 2026-02-22T21:53:43.294 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:100 IOs, 64126976 bytes 2026-02-22T21:53:43.377 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:200 IOs, 188086272 bytes 2026-02-22T21:53:43.429 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:300 IOs, 233127936 bytes 2026-02-22T21:53:43.458 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:400 IOs, 233742336 bytes 2026-02-22T21:53:43.510 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:500 IOs, 303917056 bytes 2026-02-22T21:53:43.590 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:600 IOs, 382462976 bytes 2026-02-22T21:53:43.631 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:700 IOs, 403752960 bytes 2026-02-22T21:53:43.708 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:800 IOs, 510768128 bytes 2026-02-22T21:53:43.762 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:900 IOs, 540113920 bytes 2026-02-22T21:53:43.807 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1000 IOs, 569851904 bytes 2026-02-22T21:53:43.864 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1100 IOs, 635902976 bytes 2026-02-22T21:53:43.920 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1200 IOs, 690521088 bytes 2026-02-22T21:53:43.978 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1300 IOs, 746068992 bytes 2026-02-22T21:53:44.019 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1400 IOs, 784068608 bytes 2026-02-22T21:53:44.066 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1500 IOs, 798844928 bytes 2026-02-22T21:53:44.142 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1600 IOs, 893848576 bytes 2026-02-22T21:53:44.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1700 IOs, 914509824 bytes 2026-02-22T21:53:44.231 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1800 IOs, 946127872 bytes 2026-02-22T21:53:44.275 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1900 IOs, 969124864 bytes 2026-02-22T21:53:44.303 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2000 IOs, 969724928 bytes 2026-02-22T21:53:44.352 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2100 IOs, 1005572096 bytes 2026-02-22T21:53:44.395 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2200 IOs, 1047401472 bytes 2026-02-22T21:53:44.446 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2300 IOs, 1067939840 bytes 2026-02-22T21:53:44.493 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2400 IOs, 1109122048 bytes 2026-02-22T21:53:44.538 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2500 IOs, 1139130368 bytes 2026-02-22T21:53:44.596 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2600 IOs, 1190361088 bytes 2026-02-22T21:53:44.631 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2700 IOs, 1209129984 bytes 2026-02-22T21:53:44.668 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2800 IOs, 1228086272 bytes 2026-02-22T21:53:44.701 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2900 IOs, 1239001088 bytes 2026-02-22T21:53:44.734 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3000 IOs, 1246248960 bytes 2026-02-22T21:53:44.795 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3100 IOs, 1295681536 bytes 2026-02-22T21:53:44.834 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3200 IOs, 1319415808 bytes 2026-02-22T21:53:44.884 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3300 IOs, 1354115072 bytes 2026-02-22T21:53:44.922 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3400 IOs, 1373726720 bytes 2026-02-22T21:53:44.951 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3500 IOs, 1374353408 bytes 2026-02-22T21:53:44.995 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3600 IOs, 1402471424 bytes 2026-02-22T21:53:45.052 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3700 IOs, 1500223488 bytes 2026-02-22T21:53:45.118 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3800 IOs, 1522383872 bytes 2026-02-22T21:53:45.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3900 IOs, 1620942848 bytes 2026-02-22T21:53:45.257 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4000 IOs, 1664577536 bytes 2026-02-22T21:53:45.299 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4100 IOs, 1682080768 bytes 2026-02-22T21:53:45.368 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4200 IOs, 1763201024 bytes 2026-02-22T21:53:45.431 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4300 IOs, 1823481856 bytes 2026-02-22T21:53:45.465 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4400 IOs, 1833476096 bytes 2026-02-22T21:53:45.505 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4500 IOs, 1872763904 bytes 2026-02-22T21:53:45.580 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4600 IOs, 1962085376 bytes 2026-02-22T21:53:45.619 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4700 IOs, 1977348096 bytes 2026-02-22T21:53:45.694 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4800 IOs, 2068140032 bytes 2026-02-22T21:53:45.770 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4900 IOs, 2138735616 bytes 2026-02-22T21:53:45.829 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5000 IOs, 2213239808 bytes 2026-02-22T21:53:45.878 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5100 IOs, 2244299776 bytes 2026-02-22T21:53:45.905 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5200 IOs, 2244926464 bytes 2026-02-22T21:53:45.976 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5300 IOs, 2338091008 bytes 2026-02-22T21:53:46.039 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5400 IOs, 2424760320 bytes 2026-02-22T21:53:46.099 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5500 IOs, 2462526464 bytes 2026-02-22T21:53:46.171 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5600 IOs, 2539846656 bytes 2026-02-22T21:53:46.227 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5700 IOs, 2587115520 bytes 2026-02-22T21:53:46.273 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5800 IOs, 2617438208 bytes 2026-02-22T21:53:46.301 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5900 IOs, 2618048512 bytes 2026-02-22T21:53:46.336 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6000 IOs, 2632566784 bytes 2026-02-22T21:53:46.369 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6100 IOs, 2644086784 bytes 2026-02-22T21:53:46.417 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6200 IOs, 2684238848 bytes 2026-02-22T21:53:46.457 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6300 IOs, 2718442496 bytes 2026-02-22T21:53:46.497 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6400 IOs, 2730205184 bytes 2026-02-22T21:53:46.580 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6500 IOs, 2851057664 bytes 2026-02-22T21:53:46.634 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6600 IOs, 2888782848 bytes 2026-02-22T21:53:46.683 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6700 IOs, 2943197184 bytes 2026-02-22T21:53:46.729 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6800 IOs, 2965822464 bytes 2026-02-22T21:53:46.809 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6900 IOs, 3065287680 bytes 2026-02-22T21:53:46.882 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7000 IOs, 3140421632 bytes 2026-02-22T21:53:46.928 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7100 IOs, 3165117440 bytes 2026-02-22T21:53:46.956 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7200 IOs, 3165727744 bytes 2026-02-22T21:53:47.014 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7300 IOs, 3235167232 bytes 2026-02-22T21:53:47.048 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7400 IOs, 3235803136 bytes 2026-02-22T21:53:47.097 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7500 IOs, 3296139264 bytes 2026-02-22T21:53:47.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7600 IOs, 3334858752 bytes 2026-02-22T21:53:47.224 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7700 IOs, 3389424640 bytes 2026-02-22T21:53:47.264 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7800 IOs, 3407825920 bytes 2026-02-22T21:53:47.308 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7900 IOs, 3436185600 bytes 2026-02-22T21:53:47.349 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8000 IOs, 3454889984 bytes 2026-02-22T21:53:47.427 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8100 IOs, 3544105984 bytes 2026-02-22T21:53:47.472 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8200 IOs, 3573343232 bytes 2026-02-22T21:53:47.517 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8300 IOs, 3594822656 bytes 2026-02-22T21:53:47.585 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8400 IOs, 3670774784 bytes 2026-02-22T21:53:47.619 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8500 IOs, 3683545088 bytes 2026-02-22T21:53:47.666 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8600 IOs, 3718140928 bytes 2026-02-22T21:53:47.706 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8700 IOs, 3734352896 bytes 2026-02-22T21:53:47.773 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8800 IOs, 3795427328 bytes 2026-02-22T21:53:47.834 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8900 IOs, 3857258496 bytes 2026-02-22T21:53:47.883 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9000 IOs, 3889943552 bytes 2026-02-22T21:53:47.926 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9100 IOs, 3915054080 bytes 2026-02-22T21:53:47.974 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9200 IOs, 3971052544 bytes 2026-02-22T21:53:48.015 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9300 IOs, 3971646464 bytes 2026-02-22T21:53:48.059 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9400 IOs, 4005171200 bytes 2026-02-22T21:53:48.122 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9500 IOs, 4055027712 bytes 2026-02-22T21:53:48.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9600 IOs, 4109628416 bytes 2026-02-22T21:53:48.219 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9700 IOs, 4110238720 bytes 2026-02-22T21:53:48.276 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9800 IOs, 4170775552 bytes 2026-02-22T21:53:48.327 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9900 IOs, 4214473728 bytes 2026-02-22T21:53:48.374 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10000 IOs, 4256704512 bytes 2026-02-22T21:53:48.410 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10100 IOs, 4268932096 bytes 2026-02-22T21:53:48.437 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10200 IOs, 4269532160 bytes 2026-02-22T21:53:48.496 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10300 IOs, 4333180928 bytes 2026-02-22T21:53:48.560 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10400 IOs, 4416550912 bytes 2026-02-22T21:53:48.616 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10500 IOs, 4452389888 bytes 2026-02-22T21:53:48.693 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10600 IOs, 4528132096 bytes 2026-02-22T21:53:48.744 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10700 IOs, 4594210816 bytes 2026-02-22T21:53:48.792 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10800 IOs, 4606419968 bytes 2026-02-22T21:53:48.829 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10900 IOs, 4623647744 bytes 2026-02-22T21:53:48.882 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11000 IOs, 4664375296 bytes 2026-02-22T21:53:48.915 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11100 IOs, 4672721920 bytes 2026-02-22T21:53:48.961 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11200 IOs, 4703634432 bytes 2026-02-22T21:53:49.015 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11300 IOs, 4749430784 bytes 2026-02-22T21:53:49.053 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11400 IOs, 4776849408 bytes 2026-02-22T21:53:49.096 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11500 IOs, 4803960832 bytes 2026-02-22T21:53:49.124 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11600 IOs, 4804571136 bytes 2026-02-22T21:53:49.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11700 IOs, 4840471552 bytes 2026-02-22T21:53:49.234 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11800 IOs, 4897702912 bytes 2026-02-22T21:53:49.303 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11900 IOs, 4960974848 bytes 2026-02-22T21:53:49.340 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12000 IOs, 4971553792 bytes 2026-02-22T21:53:49.418 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12100 IOs, 5070298112 bytes 2026-02-22T21:53:49.449 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12200 IOs, 5070899200 bytes 2026-02-22T21:53:49.522 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12300 IOs, 5158597632 bytes 2026-02-22T21:53:49.576 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12400 IOs, 5210786816 bytes 2026-02-22T21:53:49.613 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12500 IOs, 5218962432 bytes 2026-02-22T21:53:49.666 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12600 IOs, 5266877440 bytes 2026-02-22T21:53:49.726 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12700 IOs, 5323789312 bytes 2026-02-22T21:53:49.780 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12800 IOs, 5363165184 bytes 2026-02-22T21:53:49.840 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12900 IOs, 5397773312 bytes 2026-02-22T21:53:49.872 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13000 IOs, 5403823104 bytes 2026-02-22T21:53:49.923 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13100 IOs, 5453707264 bytes 2026-02-22T21:53:49.983 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13200 IOs, 5497370624 bytes 2026-02-22T21:53:50.018 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13300 IOs, 5513748480 bytes 2026-02-22T21:53:50.057 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13400 IOs, 5536975872 bytes 2026-02-22T21:53:50.108 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13500 IOs, 5560411136 bytes 2026-02-22T21:53:50.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13600 IOs, 5609210880 bytes 2026-02-22T21:53:50.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13700 IOs, 5625095168 bytes 2026-02-22T21:53:50.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13800 IOs, 5667109888 bytes 2026-02-22T21:53:50.314 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13900 IOs, 5720048640 bytes 2026-02-22T21:53:50.351 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14000 IOs, 5734345728 bytes 2026-02-22T21:53:50.389 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14100 IOs, 5756094464 bytes 2026-02-22T21:53:50.433 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14200 IOs, 5787600896 bytes 2026-02-22T21:53:50.493 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14300 IOs, 5855342592 bytes 2026-02-22T21:53:50.535 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14400 IOs, 5876851712 bytes 2026-02-22T21:53:50.577 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14500 IOs, 5902109696 bytes 2026-02-22T21:53:50.624 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14600 IOs, 5959076864 bytes 2026-02-22T21:53:50.666 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14700 IOs, 5959688192 bytes 2026-02-22T21:53:50.708 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14800 IOs, 5985379328 bytes 2026-02-22T21:53:50.751 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14900 IOs, 6013554688 bytes 2026-02-22T21:53:50.784 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15000 IOs, 6025197568 bytes 2026-02-22T21:53:50.829 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15100 IOs, 6043930624 bytes 2026-02-22T21:53:50.877 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15200 IOs, 6091829248 bytes 2026-02-22T21:53:50.940 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15300 IOs, 6146455552 bytes 2026-02-22T21:53:50.981 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15400 IOs, 6180417536 bytes 2026-02-22T21:53:51.031 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15500 IOs, 6220067840 bytes 2026-02-22T21:53:51.078 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15600 IOs, 6240703488 bytes 2026-02-22T21:53:51.138 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15700 IOs, 6312508416 bytes 2026-02-22T21:53:51.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15800 IOs, 6337248256 bytes 2026-02-22T21:53:51.239 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15900 IOs, 6359136256 bytes 2026-02-22T21:53:51.276 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16000 IOs, 6372186112 bytes 2026-02-22T21:53:51.309 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16100 IOs, 6382372864 bytes 2026-02-22T21:53:51.385 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16200 IOs, 6486408192 bytes 2026-02-22T21:53:51.472 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16300 IOs, 6572303360 bytes 2026-02-22T21:53:51.534 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16400 IOs, 6636326912 bytes 2026-02-22T21:53:51.562 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16500 IOs, 6636968960 bytes 2026-02-22T21:53:51.606 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16600 IOs, 6660273152 bytes 2026-02-22T21:53:51.667 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16700 IOs, 6722965504 bytes 2026-02-22T21:53:51.694 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16800 IOs, 6734922752 bytes 2026-02-22T21:53:51.729 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16900 IOs, 6736774144 bytes 2026-02-22T21:53:51.798 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17000 IOs, 6815234048 bytes 2026-02-22T21:53:51.839 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17100 IOs, 6838814720 bytes 2026-02-22T21:53:51.879 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17200 IOs, 6857989120 bytes 2026-02-22T21:53:51.956 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17300 IOs, 6944959488 bytes 2026-02-22T21:53:51.987 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17400 IOs, 6945585152 bytes 2026-02-22T21:53:52.043 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17500 IOs, 7010412544 bytes 2026-02-22T21:53:52.114 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17600 IOs, 7086752768 bytes 2026-02-22T21:53:52.170 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17700 IOs, 7123022848 bytes 2026-02-22T21:53:52.222 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17800 IOs, 7157139456 bytes 2026-02-22T21:53:52.255 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17900 IOs, 7157728256 bytes 2026-02-22T21:53:52.313 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18000 IOs, 7216875520 bytes 2026-02-22T21:53:52.345 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18100 IOs, 7220741120 bytes 2026-02-22T21:53:52.392 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18200 IOs, 7269574656 bytes 2026-02-22T21:53:52.439 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18300 IOs, 7288923136 bytes 2026-02-22T21:53:52.476 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18400 IOs, 7305149440 bytes 2026-02-22T21:53:52.510 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18500 IOs, 7316141056 bytes 2026-02-22T21:53:52.565 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18600 IOs, 7369614336 bytes 2026-02-22T21:53:52.622 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18700 IOs, 7422794752 bytes 2026-02-22T21:53:52.684 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18800 IOs, 7471587328 bytes 2026-02-22T21:53:52.728 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18900 IOs, 7514035200 bytes 2026-02-22T21:53:52.780 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19000 IOs, 7553386496 bytes 2026-02-22T21:53:52.826 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19100 IOs, 7584758784 bytes 2026-02-22T21:53:52.885 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19200 IOs, 7641393152 bytes 2026-02-22T21:53:52.966 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19300 IOs, 7721404416 bytes 2026-02-22T21:53:53.036 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19400 IOs, 7788013568 bytes 2026-02-22T21:53:53.083 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19500 IOs, 7825313792 bytes 2026-02-22T21:53:53.123 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19600 IOs, 7858531328 bytes 2026-02-22T21:53:53.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19700 IOs, 7927569408 bytes 2026-02-22T21:53:53.246 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19800 IOs, 7963279360 bytes 2026-02-22T21:53:53.291 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19900 IOs, 7979769856 bytes 2026-02-22T21:53:53.334 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20000 IOs, 8006766592 bytes 2026-02-22T21:53:53.424 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20100 IOs, 8141829120 bytes 2026-02-22T21:53:53.505 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20200 IOs, 8235520000 bytes 2026-02-22T21:53:53.541 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20300 IOs, 8248940544 bytes 2026-02-22T21:53:53.569 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20400 IOs, 8251164672 bytes 2026-02-22T21:53:53.619 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20500 IOs, 8301860864 bytes 2026-02-22T21:53:53.672 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20600 IOs, 8347819008 bytes 2026-02-22T21:53:53.735 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20700 IOs, 8411404288 bytes 2026-02-22T21:53:53.769 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20800 IOs, 8416299008 bytes 2026-02-22T21:53:53.843 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20900 IOs, 8473649152 bytes 2026-02-22T21:53:53.895 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21000 IOs, 8510737408 bytes 2026-02-22T21:53:53.951 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21100 IOs, 8573768704 bytes 2026-02-22T21:53:53.998 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21200 IOs, 8628829184 bytes 2026-02-22T21:53:54.076 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21300 IOs, 8697152512 bytes 2026-02-22T21:53:54.110 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21400 IOs, 8708857856 bytes 2026-02-22T21:53:54.159 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21500 IOs, 8749710336 bytes 2026-02-22T21:53:54.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ error_code=124 2026-02-22T21:53:54.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set -e 2026-02-22T21:53:54.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' 124 -eq 124 ']' 2026-02-22T21:53:54.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:53:54.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T21:53:54.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_37 2026-02-22T21:53:54.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image_and_enable_mirror cluster2 mirror image_37 snapshot 128M 2026-02-22T21:53:54.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:53:54.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:53:54.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:53:54.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:53:54.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_37 2026-02-22T21:53:54.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:53:54.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T21:53:54.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n snapshot ']' 2026-02-22T21:53:54.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:53:54.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image cluster2 mirror image_37 128M 2026-02-22T21:53:54.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:53:54.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:53:54.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:53:54.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:53:54.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_37 2026-02-22T21:53:54.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:53:54.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local size=128 2026-02-22T21:53:54.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n 128M ']' 2026-02-22T21:53:54.203 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ size=128M 2026-02-22T21:53:54.203 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:53:54.203 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_37 2026-02-22T21:53:54.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T21:53:54.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:53:54.233 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' image = image ']' 2026-02-22T21:53:54.233 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:53:54.233 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ enable_mirror cluster2 mirror image_37 snapshot 2026-02-22T21:53:54.233 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:53:54.233 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:53:54.233 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_37 2026-02-22T21:53:54.233 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T21:53:54.233 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_37 snapshot 2026-02-22T21:53:55.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Mirroring enabled 2026-02-22T21:53:55.173 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 info mirror/image_37 2026-02-22T21:53:55.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_37': 2026-02-22T21:53:55.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 128 MiB in 32 objects 2026-02-22T21:53:55.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T21:53:55.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 1 2026-02-22T21:53:55.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 1407f062e7db 2026-02-22T21:53:55.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.1407f062e7db 2026-02-22T21:53:55.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T21:53:55.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock 2026-02-22T21:53:55.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T21:53:55.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T21:53:55.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:53:54 2026 2026-02-22T21:53:55.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:53:54 2026 2026-02-22T21:53:55.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:53:54 2026 2026-02-22T21:53:55.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T21:53:55.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T21:53:55.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: 0ecb5a5c-9fb3-4659-bd3e-6eb17ad9c7a3 2026-02-22T21:53:55.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: true 2026-02-22T21:53:55.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n '' ']' 2026-02-22T21:53:55.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ stress_write_image cluster2 mirror image_37 2026-02-22T21:53:55.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:53:55.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:53:55.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_37 2026-02-22T21:53:55.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-22T21:53:55.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local duration=14 2026-02-22T21:53:55.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set +e 2026-02-22T21:53:55.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ timeout 14s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_37 --debug-rbd=20 --debug-journaler=20 2026-02-22T21:53:55.293 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:100 IOs, 49253376 bytes 2026-02-22T21:53:55.335 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:200 IOs, 69156864 bytes 2026-02-22T21:53:55.386 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:300 IOs, 117979136 bytes 2026-02-22T21:53:55.426 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:400 IOs, 148703232 bytes 2026-02-22T21:53:55.468 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:500 IOs, 160224256 bytes 2026-02-22T21:53:55.508 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:600 IOs, 181837824 bytes 2026-02-22T21:53:55.578 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:700 IOs, 262755328 bytes 2026-02-22T21:53:55.624 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:800 IOs, 301168640 bytes 2026-02-22T21:53:55.718 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:900 IOs, 424717312 bytes 2026-02-22T21:53:55.760 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1000 IOs, 448999424 bytes 2026-02-22T21:53:55.814 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1100 IOs, 496167936 bytes 2026-02-22T21:53:55.891 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1200 IOs, 582951936 bytes 2026-02-22T21:53:55.921 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1300 IOs, 586421248 bytes 2026-02-22T21:53:55.979 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1400 IOs, 645826560 bytes 2026-02-22T21:53:56.014 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1500 IOs, 653599744 bytes 2026-02-22T21:53:56.057 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1600 IOs, 678344704 bytes 2026-02-22T21:53:56.095 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1700 IOs, 700144640 bytes 2026-02-22T21:53:56.123 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1800 IOs, 700742656 bytes 2026-02-22T21:53:56.157 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1900 IOs, 716365824 bytes 2026-02-22T21:53:56.237 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2000 IOs, 810019840 bytes 2026-02-22T21:53:56.299 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2100 IOs, 847937536 bytes 2026-02-22T21:53:56.350 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2200 IOs, 891975680 bytes 2026-02-22T21:53:56.400 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2300 IOs, 943137792 bytes 2026-02-22T21:53:56.459 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2400 IOs, 974113792 bytes 2026-02-22T21:53:56.513 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2500 IOs, 1008340992 bytes 2026-02-22T21:53:56.546 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2600 IOs, 1016631296 bytes 2026-02-22T21:53:56.595 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2700 IOs, 1041462272 bytes 2026-02-22T21:53:56.654 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2800 IOs, 1101982720 bytes 2026-02-22T21:53:56.697 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2900 IOs, 1120496640 bytes 2026-02-22T21:53:56.751 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3000 IOs, 1160177664 bytes 2026-02-22T21:53:56.832 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3100 IOs, 1260526592 bytes 2026-02-22T21:53:56.886 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3200 IOs, 1296362496 bytes 2026-02-22T21:53:56.930 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3300 IOs, 1321792512 bytes 2026-02-22T21:53:56.973 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3400 IOs, 1355223040 bytes 2026-02-22T21:53:57.033 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3500 IOs, 1398306816 bytes 2026-02-22T21:53:57.086 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3600 IOs, 1443853312 bytes 2026-02-22T21:53:57.137 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3700 IOs, 1466366976 bytes 2026-02-22T21:53:57.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3800 IOs, 1485043712 bytes 2026-02-22T21:53:57.244 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3900 IOs, 1538549760 bytes 2026-02-22T21:53:57.298 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4000 IOs, 1593615360 bytes 2026-02-22T21:53:57.367 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4100 IOs, 1668990976 bytes 2026-02-22T21:53:57.453 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4200 IOs, 1741061120 bytes 2026-02-22T21:53:57.502 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4300 IOs, 1774467072 bytes 2026-02-22T21:53:57.573 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4400 IOs, 1847857152 bytes 2026-02-22T21:53:57.616 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4500 IOs, 1856759808 bytes 2026-02-22T21:53:57.666 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4600 IOs, 1910355968 bytes 2026-02-22T21:53:57.737 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4700 IOs, 1976193024 bytes 2026-02-22T21:53:57.786 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4800 IOs, 2001983488 bytes 2026-02-22T21:53:57.848 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4900 IOs, 2067495936 bytes 2026-02-22T21:53:57.935 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5000 IOs, 2147266560 bytes 2026-02-22T21:53:58.011 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5100 IOs, 2234100736 bytes 2026-02-22T21:53:58.051 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5200 IOs, 2262481920 bytes 2026-02-22T21:53:58.099 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5300 IOs, 2300483584 bytes 2026-02-22T21:53:58.148 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5400 IOs, 2332121088 bytes 2026-02-22T21:53:58.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5500 IOs, 2355904512 bytes 2026-02-22T21:53:58.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5600 IOs, 2383329280 bytes 2026-02-22T21:53:58.294 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5700 IOs, 2419056640 bytes 2026-02-22T21:53:58.339 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5800 IOs, 2442493952 bytes 2026-02-22T21:53:58.374 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5900 IOs, 2450707456 bytes 2026-02-22T21:53:58.421 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6000 IOs, 2480751616 bytes 2026-02-22T21:53:58.476 INFO:tasks.ceph.cluster2.osd.2.trial026.stderr:2026-02-22T21:53:58.474+0000 7f32e29d2640 -1 reset not still connected to 0x55768ebde5b0 2026-02-22T21:53:58.482 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6100 IOs, 2544331776 bytes 2026-02-22T21:53:58.518 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6200 IOs, 2568139776 bytes 2026-02-22T21:53:58.569 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6300 IOs, 2597291008 bytes 2026-02-22T21:53:58.606 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6400 IOs, 2622580736 bytes 2026-02-22T21:53:58.655 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6500 IOs, 2663409664 bytes 2026-02-22T21:53:58.720 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6600 IOs, 2724107264 bytes 2026-02-22T21:53:58.759 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6700 IOs, 2734086144 bytes 2026-02-22T21:53:58.812 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6800 IOs, 2767079424 bytes 2026-02-22T21:53:58.851 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6900 IOs, 2786094080 bytes 2026-02-22T21:53:58.895 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7000 IOs, 2805606400 bytes 2026-02-22T21:53:58.926 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7100 IOs, 2812362752 bytes 2026-02-22T21:53:58.970 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7200 IOs, 2834422784 bytes 2026-02-22T21:53:58.999 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7300 IOs, 2848267264 bytes 2026-02-22T21:53:59.042 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7400 IOs, 2857662464 bytes 2026-02-22T21:53:59.077 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7500 IOs, 2894038016 bytes 2026-02-22T21:53:59.138 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7600 IOs, 2935287808 bytes 2026-02-22T21:53:59.186 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7700 IOs, 2959752192 bytes 2026-02-22T21:53:59.247 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7800 IOs, 2998481920 bytes 2026-02-22T21:53:59.301 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7900 IOs, 3035595776 bytes 2026-02-22T21:53:59.347 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8000 IOs, 3069286400 bytes 2026-02-22T21:53:59.428 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8100 IOs, 3172054016 bytes 2026-02-22T21:53:59.498 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8200 IOs, 3248871424 bytes 2026-02-22T21:53:59.563 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8300 IOs, 3321497600 bytes 2026-02-22T21:53:59.606 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8400 IOs, 3348216832 bytes 2026-02-22T21:53:59.647 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8500 IOs, 3358580736 bytes 2026-02-22T21:53:59.690 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8600 IOs, 3373046784 bytes 2026-02-22T21:53:59.736 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8700 IOs, 3411657728 bytes 2026-02-22T21:53:59.772 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8800 IOs, 3435827200 bytes 2026-02-22T21:53:59.831 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8900 IOs, 3474001920 bytes 2026-02-22T21:53:59.881 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9000 IOs, 3514211328 bytes 2026-02-22T21:53:59.955 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9100 IOs, 3579666432 bytes 2026-02-22T21:54:00.028 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9200 IOs, 3664363520 bytes 2026-02-22T21:54:00.093 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9300 IOs, 3713011712 bytes 2026-02-22T21:54:00.138 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9400 IOs, 3755269120 bytes 2026-02-22T21:54:00.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9500 IOs, 3857845248 bytes 2026-02-22T21:54:00.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9600 IOs, 3868118016 bytes 2026-02-22T21:54:00.318 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9700 IOs, 3899478016 bytes 2026-02-22T21:54:00.374 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9800 IOs, 3941553152 bytes 2026-02-22T21:54:00.428 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9900 IOs, 3993000960 bytes 2026-02-22T21:54:00.486 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10000 IOs, 4038233088 bytes 2026-02-22T21:54:00.539 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10100 IOs, 4075302912 bytes 2026-02-22T21:54:00.571 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10200 IOs, 4097807360 bytes 2026-02-22T21:54:00.629 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10300 IOs, 4135885824 bytes 2026-02-22T21:54:00.690 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10400 IOs, 4219684864 bytes 2026-02-22T21:54:00.741 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10500 IOs, 4245134336 bytes 2026-02-22T21:54:00.816 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10600 IOs, 4336193536 bytes 2026-02-22T21:54:00.875 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10700 IOs, 4402150400 bytes 2026-02-22T21:54:00.933 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10800 IOs, 4441106432 bytes 2026-02-22T21:54:00.995 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10900 IOs, 4493501440 bytes 2026-02-22T21:54:01.045 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11000 IOs, 4562485248 bytes 2026-02-22T21:54:01.121 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11100 IOs, 4631222272 bytes 2026-02-22T21:54:01.162 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11200 IOs, 4652651520 bytes 2026-02-22T21:54:01.215 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11300 IOs, 4689982464 bytes 2026-02-22T21:54:01.282 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11400 IOs, 4740009984 bytes 2026-02-22T21:54:01.325 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11500 IOs, 4762772480 bytes 2026-02-22T21:54:01.388 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11600 IOs, 4850014208 bytes 2026-02-22T21:54:01.455 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11700 IOs, 4920378368 bytes 2026-02-22T21:54:01.515 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11800 IOs, 4957264896 bytes 2026-02-22T21:54:01.565 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11900 IOs, 5007093760 bytes 2026-02-22T21:54:01.627 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12000 IOs, 5075297280 bytes 2026-02-22T21:54:01.704 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12100 IOs, 5142367232 bytes 2026-02-22T21:54:01.755 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12200 IOs, 5180500992 bytes 2026-02-22T21:54:01.817 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12300 IOs, 5253381120 bytes 2026-02-22T21:54:01.854 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12400 IOs, 5267184640 bytes 2026-02-22T21:54:01.917 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12500 IOs, 5344380928 bytes 2026-02-22T21:54:01.978 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12600 IOs, 5386232832 bytes 2026-02-22T21:54:02.027 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12700 IOs, 5431437312 bytes 2026-02-22T21:54:02.065 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12800 IOs, 5449871360 bytes 2026-02-22T21:54:02.137 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12900 IOs, 5529187328 bytes 2026-02-22T21:54:02.184 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13000 IOs, 5554198528 bytes 2026-02-22T21:54:02.235 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13100 IOs, 5599632384 bytes 2026-02-22T21:54:02.289 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13200 IOs, 5636902912 bytes 2026-02-22T21:54:02.335 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13300 IOs, 5666284544 bytes 2026-02-22T21:54:02.383 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13400 IOs, 5693563904 bytes 2026-02-22T21:54:02.430 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13500 IOs, 5717604352 bytes 2026-02-22T21:54:02.465 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13600 IOs, 5733707776 bytes 2026-02-22T21:54:02.500 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13700 IOs, 5745517568 bytes 2026-02-22T21:54:02.577 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13800 IOs, 5833475072 bytes 2026-02-22T21:54:02.632 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13900 IOs, 5880377344 bytes 2026-02-22T21:54:02.688 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14000 IOs, 5928197120 bytes 2026-02-22T21:54:02.733 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14100 IOs, 5957492736 bytes 2026-02-22T21:54:02.776 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14200 IOs, 5983033344 bytes 2026-02-22T21:54:02.841 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14300 IOs, 6059705344 bytes 2026-02-22T21:54:02.923 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14400 IOs, 6181565440 bytes 2026-02-22T21:54:02.975 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14500 IOs, 6200808448 bytes 2026-02-22T21:54:03.004 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14600 IOs, 6201445376 bytes 2026-02-22T21:54:03.035 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14700 IOs, 6206174208 bytes 2026-02-22T21:54:03.064 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14800 IOs, 6206826496 bytes 2026-02-22T21:54:03.095 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14900 IOs, 6207465472 bytes 2026-02-22T21:54:03.133 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15000 IOs, 6223604736 bytes 2026-02-22T21:54:03.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15100 IOs, 6237648896 bytes 2026-02-22T21:54:03.249 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15200 IOs, 6325554176 bytes 2026-02-22T21:54:03.308 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15300 IOs, 6368687104 bytes 2026-02-22T21:54:03.354 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15400 IOs, 6403184640 bytes 2026-02-22T21:54:03.394 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15500 IOs, 6415373312 bytes 2026-02-22T21:54:03.438 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15600 IOs, 6442971136 bytes 2026-02-22T21:54:03.488 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15700 IOs, 6470759424 bytes 2026-02-22T21:54:03.526 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15800 IOs, 6485810176 bytes 2026-02-22T21:54:03.585 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15900 IOs, 6551665664 bytes 2026-02-22T21:54:03.636 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16000 IOs, 6581030912 bytes 2026-02-22T21:54:03.676 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16100 IOs, 6593958912 bytes 2026-02-22T21:54:03.728 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16200 IOs, 6641763328 bytes 2026-02-22T21:54:03.795 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16300 IOs, 6727273472 bytes 2026-02-22T21:54:03.853 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16400 IOs, 6767681536 bytes 2026-02-22T21:54:03.897 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16500 IOs, 6795487232 bytes 2026-02-22T21:54:03.943 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16600 IOs, 6826224640 bytes 2026-02-22T21:54:03.996 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16700 IOs, 6847628288 bytes 2026-02-22T21:54:04.038 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16800 IOs, 6877649920 bytes 2026-02-22T21:54:04.082 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16900 IOs, 6897078272 bytes 2026-02-22T21:54:04.135 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17000 IOs, 6936828928 bytes 2026-02-22T21:54:04.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17100 IOs, 6979663872 bytes 2026-02-22T21:54:04.245 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17200 IOs, 7013936128 bytes 2026-02-22T21:54:04.289 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17300 IOs, 7033422848 bytes 2026-02-22T21:54:04.343 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17400 IOs, 7074575360 bytes 2026-02-22T21:54:04.401 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17500 IOs, 7126172672 bytes 2026-02-22T21:54:04.458 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17600 IOs, 7164561408 bytes 2026-02-22T21:54:04.489 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17700 IOs, 7165181952 bytes 2026-02-22T21:54:04.549 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17800 IOs, 7220498432 bytes 2026-02-22T21:54:04.786 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17900 IOs, 7221645312 bytes 2026-02-22T21:54:04.828 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18000 IOs, 7240863744 bytes 2026-02-22T21:54:04.893 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18100 IOs, 7307939840 bytes 2026-02-22T21:54:04.965 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18200 IOs, 7376055296 bytes 2026-02-22T21:54:05.031 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18300 IOs, 7431909376 bytes 2026-02-22T21:54:05.110 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18400 IOs, 7522938880 bytes 2026-02-22T21:54:05.169 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18500 IOs, 7580457984 bytes 2026-02-22T21:54:05.220 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18600 IOs, 7612937216 bytes 2026-02-22T21:54:05.262 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18700 IOs, 7633849344 bytes 2026-02-22T21:54:05.348 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18800 IOs, 7721525248 bytes 2026-02-22T21:54:05.383 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18900 IOs, 7726187520 bytes 2026-02-22T21:54:05.432 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19000 IOs, 7767364608 bytes 2026-02-22T21:54:05.465 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19100 IOs, 7770915840 bytes 2026-02-22T21:54:05.518 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19200 IOs, 7812869120 bytes 2026-02-22T21:54:05.564 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19300 IOs, 7838560256 bytes 2026-02-22T21:54:05.614 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19400 IOs, 7864384512 bytes 2026-02-22T21:54:05.663 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19500 IOs, 7890550784 bytes 2026-02-22T21:54:05.759 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19600 IOs, 8065273856 bytes 2026-02-22T21:54:05.824 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19700 IOs, 8087664640 bytes 2026-02-22T21:54:05.867 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19800 IOs, 8116467712 bytes 2026-02-22T21:54:05.897 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19900 IOs, 8141726720 bytes 2026-02-22T21:54:05.950 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20000 IOs, 8155516928 bytes 2026-02-22T21:54:06.002 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20100 IOs, 8196390912 bytes 2026-02-22T21:54:06.065 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20200 IOs, 8259617792 bytes 2026-02-22T21:54:06.121 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20300 IOs, 8298377216 bytes 2026-02-22T21:54:06.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20400 IOs, 8336757760 bytes 2026-02-22T21:54:06.265 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20500 IOs, 8438597632 bytes 2026-02-22T21:54:06.322 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20600 IOs, 8476881920 bytes 2026-02-22T21:54:06.365 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20700 IOs, 8482636800 bytes 2026-02-22T21:54:06.417 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20800 IOs, 8528528384 bytes 2026-02-22T21:54:06.480 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20900 IOs, 8569283584 bytes 2026-02-22T21:54:06.527 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21000 IOs, 8598142976 bytes 2026-02-22T21:54:06.569 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21100 IOs, 8612846592 bytes 2026-02-22T21:54:06.638 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21200 IOs, 8678828032 bytes 2026-02-22T21:54:06.695 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21300 IOs, 8724687872 bytes 2026-02-22T21:54:06.735 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21400 IOs, 8742839296 bytes 2026-02-22T21:54:06.813 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21500 IOs, 8818241536 bytes 2026-02-22T21:54:06.848 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21600 IOs, 8823677952 bytes 2026-02-22T21:54:07.080 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21700 IOs, 8832822272 bytes 2026-02-22T21:54:07.136 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21800 IOs, 8871048192 bytes 2026-02-22T21:54:07.183 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21900 IOs, 8896493568 bytes 2026-02-22T21:54:07.256 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22000 IOs, 8962813952 bytes 2026-02-22T21:54:07.317 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22100 IOs, 9000446976 bytes 2026-02-22T21:54:07.377 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22200 IOs, 9045837824 bytes 2026-02-22T21:54:07.427 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22300 IOs, 9081307136 bytes 2026-02-22T21:54:07.478 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22400 IOs, 9114737664 bytes 2026-02-22T21:54:07.523 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22500 IOs, 9133611008 bytes 2026-02-22T21:54:07.561 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22600 IOs, 9141299200 bytes 2026-02-22T21:54:07.633 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22700 IOs, 9209830400 bytes 2026-02-22T21:54:07.673 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22800 IOs, 9231188992 bytes 2026-02-22T21:54:07.934 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22900 IOs, 9289265152 bytes 2026-02-22T21:54:07.989 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23000 IOs, 9320680448 bytes 2026-02-22T21:54:08.042 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23100 IOs, 9347295232 bytes 2026-02-22T21:54:08.129 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23200 IOs, 9438747648 bytes 2026-02-22T21:54:08.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23300 IOs, 9522070528 bytes 2026-02-22T21:54:08.250 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23400 IOs, 9551172608 bytes 2026-02-22T21:54:08.305 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23500 IOs, 9608747008 bytes 2026-02-22T21:54:08.363 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23600 IOs, 9630475264 bytes 2026-02-22T21:54:08.398 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23700 IOs, 9635420160 bytes 2026-02-22T21:54:08.483 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23800 IOs, 9730852864 bytes 2026-02-22T21:54:08.546 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23900 IOs, 9783097344 bytes 2026-02-22T21:54:08.598 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24000 IOs, 9813387264 bytes 2026-02-22T21:54:08.656 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24100 IOs, 9864671232 bytes 2026-02-22T21:54:08.720 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24200 IOs, 9924333568 bytes 2026-02-22T21:54:08.774 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24300 IOs, 9961896960 bytes 2026-02-22T21:54:08.821 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24400 IOs, 9991314432 bytes 2026-02-22T21:54:08.883 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24500 IOs, 10042338304 bytes 2026-02-22T21:54:08.926 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24600 IOs, 10060814336 bytes 2026-02-22T21:54:08.966 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24700 IOs, 10070955008 bytes 2026-02-22T21:54:09.036 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24800 IOs, 10152808448 bytes 2026-02-22T21:54:09.086 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24900 IOs, 10157651968 bytes 2026-02-22T21:54:09.145 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25000 IOs, 10218360832 bytes 2026-02-22T21:54:09.188 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25100 IOs, 10221432832 bytes 2026-02-22T21:54:09.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ error_code=124 2026-02-22T21:54:09.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set -e 2026-02-22T21:54:09.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' 124 -eq 124 ']' 2026-02-22T21:54:09.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:54:09.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T21:54:09.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_38 2026-02-22T21:54:09.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image_and_enable_mirror cluster2 mirror image_38 snapshot 128M 2026-02-22T21:54:09.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:54:09.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:54:09.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:54:09.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:54:09.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_38 2026-02-22T21:54:09.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:54:09.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T21:54:09.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n snapshot ']' 2026-02-22T21:54:09.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:54:09.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image cluster2 mirror image_38 128M 2026-02-22T21:54:09.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:54:09.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:54:09.209 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:54:09.209 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:54:09.209 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_38 2026-02-22T21:54:09.209 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:54:09.209 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local size=128 2026-02-22T21:54:09.209 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n 128M ']' 2026-02-22T21:54:09.209 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ size=128M 2026-02-22T21:54:09.209 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:54:09.209 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_38 2026-02-22T21:54:09.239 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T21:54:09.239 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:54:09.239 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' image = image ']' 2026-02-22T21:54:09.239 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:54:09.239 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ enable_mirror cluster2 mirror image_38 snapshot 2026-02-22T21:54:09.239 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:54:09.240 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:54:09.240 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_38 2026-02-22T21:54:09.240 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T21:54:09.240 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_38 snapshot 2026-02-22T21:54:10.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Mirroring enabled 2026-02-22T21:54:10.183 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 info mirror/image_38 2026-02-22T21:54:10.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_38': 2026-02-22T21:54:10.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 128 MiB in 32 objects 2026-02-22T21:54:10.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T21:54:10.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 1 2026-02-22T21:54:10.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 14138b9b6a93 2026-02-22T21:54:10.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.14138b9b6a93 2026-02-22T21:54:10.209 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T21:54:10.209 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock 2026-02-22T21:54:10.209 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T21:54:10.209 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T21:54:10.209 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:54:09 2026 2026-02-22T21:54:10.209 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:54:09 2026 2026-02-22T21:54:10.209 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:54:09 2026 2026-02-22T21:54:10.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T21:54:10.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T21:54:10.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: db20ecda-1e60-4b4e-b471-d368349418c1 2026-02-22T21:54:10.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: true 2026-02-22T21:54:10.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n '' ']' 2026-02-22T21:54:10.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ stress_write_image cluster2 mirror image_38 2026-02-22T21:54:10.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:54:10.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:54:10.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_38 2026-02-22T21:54:10.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-22T21:54:10.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local duration=9 2026-02-22T21:54:10.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set +e 2026-02-22T21:54:10.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ timeout 9s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_38 --debug-rbd=20 --debug-journaler=20 2026-02-22T21:54:10.308 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:100 IOs, 74228736 bytes 2026-02-22T21:54:10.364 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:200 IOs, 113136640 bytes 2026-02-22T21:54:10.405 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:300 IOs, 126332928 bytes 2026-02-22T21:54:10.449 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:400 IOs, 155679744 bytes 2026-02-22T21:54:10.504 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:500 IOs, 229866496 bytes 2026-02-22T21:54:10.553 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:600 IOs, 254260224 bytes 2026-02-22T21:54:10.605 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:700 IOs, 307928064 bytes 2026-02-22T21:54:10.663 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:800 IOs, 372452352 bytes 2026-02-22T21:54:10.698 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:900 IOs, 385974272 bytes 2026-02-22T21:54:10.749 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1000 IOs, 430496768 bytes 2026-02-22T21:54:10.802 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1100 IOs, 470854656 bytes 2026-02-22T21:54:10.857 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1200 IOs, 533677056 bytes 2026-02-22T21:54:10.892 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1300 IOs, 543443968 bytes 2026-02-22T21:54:10.935 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1400 IOs, 574067712 bytes 2026-02-22T21:54:10.997 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1500 IOs, 638999552 bytes 2026-02-22T21:54:11.054 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1600 IOs, 680148992 bytes 2026-02-22T21:54:11.102 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1700 IOs, 716813312 bytes 2026-02-22T21:54:11.152 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1800 IOs, 770809856 bytes 2026-02-22T21:54:11.230 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1900 IOs, 872790016 bytes 2026-02-22T21:54:11.294 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2000 IOs, 935170048 bytes 2026-02-22T21:54:11.352 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2100 IOs, 973147136 bytes 2026-02-22T21:54:11.402 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2200 IOs, 1004158976 bytes 2026-02-22T21:54:11.444 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2300 IOs, 1037188096 bytes 2026-02-22T21:54:11.501 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2400 IOs, 1107525632 bytes 2026-02-22T21:54:11.546 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2500 IOs, 1126657024 bytes 2026-02-22T21:54:11.581 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2600 IOs, 1147015168 bytes 2026-02-22T21:54:11.629 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2700 IOs, 1198169088 bytes 2026-02-22T21:54:11.690 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2800 IOs, 1238290432 bytes 2026-02-22T21:54:11.724 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2900 IOs, 1274324992 bytes 2026-02-22T21:54:11.791 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3000 IOs, 1325057024 bytes 2026-02-22T21:54:11.836 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3100 IOs, 1370889216 bytes 2026-02-22T21:54:11.900 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3200 IOs, 1439002624 bytes 2026-02-22T21:54:11.951 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3300 IOs, 1484978176 bytes 2026-02-22T21:54:12.001 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3400 IOs, 1519858688 bytes 2026-02-22T21:54:12.054 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3500 IOs, 1565448192 bytes 2026-02-22T21:54:12.107 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3600 IOs, 1630882816 bytes 2026-02-22T21:54:12.161 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3700 IOs, 1665108992 bytes 2026-02-22T21:54:12.219 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3800 IOs, 1709433856 bytes 2026-02-22T21:54:12.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3900 IOs, 1789184000 bytes 2026-02-22T21:54:12.373 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4000 IOs, 1905510400 bytes 2026-02-22T21:54:12.428 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4100 IOs, 1951725568 bytes 2026-02-22T21:54:12.704 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4200 IOs, 2059760640 bytes 2026-02-22T21:54:12.755 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4300 IOs, 2100326400 bytes 2026-02-22T21:54:12.811 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4400 IOs, 2160297984 bytes 2026-02-22T21:54:12.856 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4500 IOs, 2171658240 bytes 2026-02-22T21:54:12.907 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4600 IOs, 2220045312 bytes 2026-02-22T21:54:12.959 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4700 IOs, 2258587648 bytes 2026-02-22T21:54:13.004 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4800 IOs, 2291861504 bytes 2026-02-22T21:54:13.070 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4900 IOs, 2346624000 bytes 2026-02-22T21:54:13.118 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5000 IOs, 2389942272 bytes 2026-02-22T21:54:13.162 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5100 IOs, 2424290304 bytes 2026-02-22T21:54:13.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5200 IOs, 2446758912 bytes 2026-02-22T21:54:13.255 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5300 IOs, 2500406272 bytes 2026-02-22T21:54:13.298 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5400 IOs, 2529228800 bytes 2026-02-22T21:54:13.338 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5500 IOs, 2554439680 bytes 2026-02-22T21:54:13.389 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5600 IOs, 2583035904 bytes 2026-02-22T21:54:13.442 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5700 IOs, 2623712256 bytes 2026-02-22T21:54:13.510 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5800 IOs, 2702202880 bytes 2026-02-22T21:54:13.554 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5900 IOs, 2730402816 bytes 2026-02-22T21:54:13.606 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6000 IOs, 2783589376 bytes 2026-02-22T21:54:13.673 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6100 IOs, 2844516352 bytes 2026-02-22T21:54:13.733 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6200 IOs, 2897617920 bytes 2026-02-22T21:54:13.764 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6300 IOs, 2901282816 bytes 2026-02-22T21:54:13.811 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6400 IOs, 2941635584 bytes 2026-02-22T21:54:13.838 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6500 IOs, 2943419392 bytes 2026-02-22T21:54:13.909 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6600 IOs, 3019777024 bytes 2026-02-22T21:54:13.944 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6700 IOs, 3047854080 bytes 2026-02-22T21:54:13.990 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6800 IOs, 3071272960 bytes 2026-02-22T21:54:14.027 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6900 IOs, 3087589376 bytes 2026-02-22T21:54:14.082 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7000 IOs, 3138801664 bytes 2026-02-22T21:54:14.125 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7100 IOs, 3171326976 bytes 2026-02-22T21:54:14.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7200 IOs, 3191887872 bytes 2026-02-22T21:54:14.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7300 IOs, 3213307904 bytes 2026-02-22T21:54:14.269 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7400 IOs, 3287512064 bytes 2026-02-22T21:54:14.317 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7500 IOs, 3330893824 bytes 2026-02-22T21:54:14.366 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7600 IOs, 3362699264 bytes 2026-02-22T21:54:14.405 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7700 IOs, 3379356672 bytes 2026-02-22T21:54:14.443 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7800 IOs, 3385952256 bytes 2026-02-22T21:54:14.491 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7900 IOs, 3422864384 bytes 2026-02-22T21:54:14.539 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8000 IOs, 3452803072 bytes 2026-02-22T21:54:14.599 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8100 IOs, 3502665728 bytes 2026-02-22T21:54:14.625 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8200 IOs, 3503270912 bytes 2026-02-22T21:54:14.653 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8300 IOs, 3503883264 bytes 2026-02-22T21:54:14.681 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8400 IOs, 3504513024 bytes 2026-02-22T21:54:14.756 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8500 IOs, 3605042176 bytes 2026-02-22T21:54:14.846 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8600 IOs, 3726613504 bytes 2026-02-22T21:54:14.895 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8700 IOs, 3766713344 bytes 2026-02-22T21:54:14.932 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8800 IOs, 3783899136 bytes 2026-02-22T21:54:14.973 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8900 IOs, 3829359616 bytes 2026-02-22T21:54:15.019 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9000 IOs, 3837492224 bytes 2026-02-22T21:54:15.085 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9100 IOs, 3893455872 bytes 2026-02-22T21:54:15.148 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9200 IOs, 3963665408 bytes 2026-02-22T21:54:15.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9300 IOs, 4005784576 bytes 2026-02-22T21:54:15.264 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9400 IOs, 4071884800 bytes 2026-02-22T21:54:15.300 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9500 IOs, 4087142400 bytes 2026-02-22T21:54:15.353 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9600 IOs, 4139067392 bytes 2026-02-22T21:54:15.412 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9700 IOs, 4176821248 bytes 2026-02-22T21:54:15.464 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9800 IOs, 4222914560 bytes 2026-02-22T21:54:15.528 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9900 IOs, 4264437760 bytes 2026-02-22T21:54:15.584 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10000 IOs, 4319455232 bytes 2026-02-22T21:54:15.635 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10100 IOs, 4361576448 bytes 2026-02-22T21:54:15.711 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10200 IOs, 4456005632 bytes 2026-02-22T21:54:15.759 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10300 IOs, 4484462592 bytes 2026-02-22T21:54:15.797 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10400 IOs, 4503996416 bytes 2026-02-22T21:54:15.842 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10500 IOs, 4536636416 bytes 2026-02-22T21:54:15.892 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10600 IOs, 4572645376 bytes 2026-02-22T21:54:15.924 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10700 IOs, 4586682368 bytes 2026-02-22T21:54:15.985 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10800 IOs, 4640557056 bytes 2026-02-22T21:54:16.036 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10900 IOs, 4686129152 bytes 2026-02-22T21:54:16.081 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11000 IOs, 4707343360 bytes 2026-02-22T21:54:16.133 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11100 IOs, 4752631808 bytes 2026-02-22T21:54:16.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11200 IOs, 4775578624 bytes 2026-02-22T21:54:16.218 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11300 IOs, 4794705920 bytes 2026-02-22T21:54:16.247 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11400 IOs, 4800887808 bytes 2026-02-22T21:54:16.282 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11500 IOs, 4814900224 bytes 2026-02-22T21:54:16.376 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11600 IOs, 4942965760 bytes 2026-02-22T21:54:16.426 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11700 IOs, 4970761216 bytes 2026-02-22T21:54:16.485 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11800 IOs, 5028485120 bytes 2026-02-22T21:54:16.544 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11900 IOs, 5088321536 bytes 2026-02-22T21:54:16.587 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12000 IOs, 5104357376 bytes 2026-02-22T21:54:16.631 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12100 IOs, 5131433984 bytes 2026-02-22T21:54:16.664 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12200 IOs, 5138793472 bytes 2026-02-22T21:54:16.712 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12300 IOs, 5179623424 bytes 2026-02-22T21:54:16.751 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12400 IOs, 5199433728 bytes 2026-02-22T21:54:16.819 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12500 IOs, 5280715776 bytes 2026-02-22T21:54:16.865 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12600 IOs, 5314900992 bytes 2026-02-22T21:54:16.921 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12700 IOs, 5363438592 bytes 2026-02-22T21:54:16.991 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12800 IOs, 5443999744 bytes 2026-02-22T21:54:17.064 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12900 IOs, 5539342336 bytes 2026-02-22T21:54:17.096 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13000 IOs, 5554268160 bytes 2026-02-22T21:54:17.175 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13100 IOs, 5638945792 bytes 2026-02-22T21:54:17.214 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13200 IOs, 5661913088 bytes 2026-02-22T21:54:17.263 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13300 IOs, 5706432512 bytes 2026-02-22T21:54:17.331 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13400 IOs, 5788002304 bytes 2026-02-22T21:54:17.382 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13500 IOs, 5823542272 bytes 2026-02-22T21:54:17.432 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13600 IOs, 5859213312 bytes 2026-02-22T21:54:17.480 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13700 IOs, 5884365824 bytes 2026-02-22T21:54:17.536 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13800 IOs, 5940498432 bytes 2026-02-22T21:54:17.587 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13900 IOs, 5972731904 bytes 2026-02-22T21:54:17.624 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14000 IOs, 5993621504 bytes 2026-02-22T21:54:17.688 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14100 IOs, 6076886016 bytes 2026-02-22T21:54:17.748 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14200 IOs, 6113101824 bytes 2026-02-22T21:54:17.792 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14300 IOs, 6164779008 bytes 2026-02-22T21:54:17.832 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14400 IOs, 6168236032 bytes 2026-02-22T21:54:17.891 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14500 IOs, 6224878592 bytes 2026-02-22T21:54:17.923 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14600 IOs, 6241454080 bytes 2026-02-22T21:54:17.969 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14700 IOs, 6273153024 bytes 2026-02-22T21:54:18.021 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14800 IOs, 6314189824 bytes 2026-02-22T21:54:18.053 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14900 IOs, 6321800192 bytes 2026-02-22T21:54:18.122 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15000 IOs, 6380106752 bytes 2026-02-22T21:54:18.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15100 IOs, 6432774144 bytes 2026-02-22T21:54:18.249 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15200 IOs, 6519488512 bytes 2026-02-22T21:54:18.305 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15300 IOs, 6565246976 bytes 2026-02-22T21:54:18.356 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15400 IOs, 6595598336 bytes 2026-02-22T21:54:18.412 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15500 IOs, 6649223168 bytes 2026-02-22T21:54:18.479 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15600 IOs, 6714969088 bytes 2026-02-22T21:54:18.541 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15700 IOs, 6767703040 bytes 2026-02-22T21:54:18.581 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15800 IOs, 6787553280 bytes 2026-02-22T21:54:18.618 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15900 IOs, 6801783808 bytes 2026-02-22T21:54:18.657 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16000 IOs, 6815179776 bytes 2026-02-22T21:54:18.685 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16100 IOs, 6816444416 bytes 2026-02-22T21:54:18.726 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16200 IOs, 6839950336 bytes 2026-02-22T21:54:18.792 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16300 IOs, 6908182528 bytes 2026-02-22T21:54:18.854 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16400 IOs, 7001994240 bytes 2026-02-22T21:54:18.911 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16500 IOs, 7037849600 bytes 2026-02-22T21:54:18.964 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16600 IOs, 7082257408 bytes 2026-02-22T21:54:19.019 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16700 IOs, 7137677312 bytes 2026-02-22T21:54:19.049 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16800 IOs, 7138279424 bytes 2026-02-22T21:54:19.103 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16900 IOs, 7192180736 bytes 2026-02-22T21:54:19.173 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17000 IOs, 7285655552 bytes 2026-02-22T21:54:19.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ error_code=124 2026-02-22T21:54:19.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set -e 2026-02-22T21:54:19.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' 124 -eq 124 ']' 2026-02-22T21:54:19.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:54:19.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T21:54:19.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_39 2026-02-22T21:54:19.218 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image_and_enable_mirror cluster2 mirror image_39 snapshot 128M 2026-02-22T21:54:19.218 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:54:19.218 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:54:19.218 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:54:19.218 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:54:19.218 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_39 2026-02-22T21:54:19.218 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:54:19.218 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T21:54:19.218 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n snapshot ']' 2026-02-22T21:54:19.218 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:54:19.218 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image cluster2 mirror image_39 128M 2026-02-22T21:54:19.218 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:54:19.219 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:54:19.219 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:54:19.219 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:54:19.219 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_39 2026-02-22T21:54:19.219 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:54:19.219 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local size=128 2026-02-22T21:54:19.219 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n 128M ']' 2026-02-22T21:54:19.219 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ size=128M 2026-02-22T21:54:19.219 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:54:19.219 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_39 2026-02-22T21:54:19.247 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T21:54:19.247 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:54:19.247 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' image = image ']' 2026-02-22T21:54:19.247 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:54:19.247 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ enable_mirror cluster2 mirror image_39 snapshot 2026-02-22T21:54:19.247 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:54:19.247 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:54:19.248 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_39 2026-02-22T21:54:19.248 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T21:54:19.248 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_39 snapshot 2026-02-22T21:54:20.138 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Mirroring enabled 2026-02-22T21:54:20.143 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 info mirror/image_39 2026-02-22T21:54:20.166 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_39': 2026-02-22T21:54:20.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 128 MiB in 32 objects 2026-02-22T21:54:20.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T21:54:20.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 1 2026-02-22T21:54:20.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 141f3472c42f 2026-02-22T21:54:20.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.141f3472c42f 2026-02-22T21:54:20.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T21:54:20.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock 2026-02-22T21:54:20.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T21:54:20.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T21:54:20.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:54:19 2026 2026-02-22T21:54:20.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:54:19 2026 2026-02-22T21:54:20.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:54:19 2026 2026-02-22T21:54:20.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T21:54:20.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T21:54:20.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: 92432590-5c9b-4a0c-b269-4ed3db4a7657 2026-02-22T21:54:20.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: true 2026-02-22T21:54:20.169 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n '' ']' 2026-02-22T21:54:20.169 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ stress_write_image cluster2 mirror image_39 2026-02-22T21:54:20.169 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:54:20.169 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:54:20.169 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_39 2026-02-22T21:54:20.170 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-22T21:54:20.170 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local duration=7 2026-02-22T21:54:20.170 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set +e 2026-02-22T21:54:20.170 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ timeout 7s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_39 --debug-rbd=20 --debug-journaler=20 2026-02-22T21:54:20.258 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:100 IOs, 73589760 bytes 2026-02-22T21:54:20.324 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:200 IOs, 145290240 bytes 2026-02-22T21:54:20.392 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:300 IOs, 222739456 bytes 2026-02-22T21:54:20.427 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:400 IOs, 229282816 bytes 2026-02-22T21:54:20.468 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:500 IOs, 251532288 bytes 2026-02-22T21:54:20.526 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:600 IOs, 314684416 bytes 2026-02-22T21:54:20.589 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:700 IOs, 381809664 bytes 2026-02-22T21:54:20.669 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:800 IOs, 476425216 bytes 2026-02-22T21:54:20.698 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:900 IOs, 477035520 bytes 2026-02-22T21:54:20.752 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1000 IOs, 521522176 bytes 2026-02-22T21:54:20.816 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1100 IOs, 585453568 bytes 2026-02-22T21:54:20.869 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1200 IOs, 632012800 bytes 2026-02-22T21:54:20.908 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1300 IOs, 662394880 bytes 2026-02-22T21:54:20.951 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1400 IOs, 692941824 bytes 2026-02-22T21:54:20.998 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1500 IOs, 732370944 bytes 2026-02-22T21:54:21.032 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1600 IOs, 745647104 bytes 2026-02-22T21:54:21.083 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1700 IOs, 800771072 bytes 2026-02-22T21:54:21.150 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1800 IOs, 844113920 bytes 2026-02-22T21:54:21.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1900 IOs, 894354432 bytes 2026-02-22T21:54:21.246 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2000 IOs, 939280384 bytes 2026-02-22T21:54:21.294 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2100 IOs, 968636416 bytes 2026-02-22T21:54:21.337 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2200 IOs, 1004771328 bytes 2026-02-22T21:54:21.394 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2300 IOs, 1049846784 bytes 2026-02-22T21:54:21.447 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2400 IOs, 1096444928 bytes 2026-02-22T21:54:21.533 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2500 IOs, 1194272768 bytes 2026-02-22T21:54:21.561 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2600 IOs, 1194889216 bytes 2026-02-22T21:54:21.612 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2700 IOs, 1246226432 bytes 2026-02-22T21:54:21.654 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2800 IOs, 1283137536 bytes 2026-02-22T21:54:21.712 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2900 IOs, 1332844544 bytes 2026-02-22T21:54:21.760 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3000 IOs, 1368092672 bytes 2026-02-22T21:54:21.797 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3100 IOs, 1398243328 bytes 2026-02-22T21:54:21.861 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3200 IOs, 1450744832 bytes 2026-02-22T21:54:21.901 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3300 IOs, 1483683840 bytes 2026-02-22T21:54:21.974 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3400 IOs, 1548694528 bytes 2026-02-22T21:54:22.027 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3500 IOs, 1603371008 bytes 2026-02-22T21:54:22.085 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3600 IOs, 1693943808 bytes 2026-02-22T21:54:22.151 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3700 IOs, 1720051712 bytes 2026-02-22T21:54:22.195 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3800 IOs, 1737446400 bytes 2026-02-22T21:54:22.246 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3900 IOs, 1770506240 bytes 2026-02-22T21:54:22.282 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4000 IOs, 1787858944 bytes 2026-02-22T21:54:22.319 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4100 IOs, 1805551616 bytes 2026-02-22T21:54:22.383 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4200 IOs, 1874834432 bytes 2026-02-22T21:54:22.427 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4300 IOs, 1899227136 bytes 2026-02-22T21:54:22.478 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4400 IOs, 1922317312 bytes 2026-02-22T21:54:22.531 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4500 IOs, 1974920192 bytes 2026-02-22T21:54:22.583 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4600 IOs, 2001845248 bytes 2026-02-22T21:54:22.620 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4700 IOs, 2025129984 bytes 2026-02-22T21:54:22.657 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4800 IOs, 2042509312 bytes 2026-02-22T21:54:22.693 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4900 IOs, 2054065152 bytes 2026-02-22T21:54:22.767 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5000 IOs, 2151347200 bytes 2026-02-22T21:54:22.848 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5100 IOs, 2244342784 bytes 2026-02-22T21:54:22.896 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5200 IOs, 2310480896 bytes 2026-02-22T21:54:22.944 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5300 IOs, 2325049344 bytes 2026-02-22T21:54:23.021 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5400 IOs, 2427025408 bytes 2026-02-22T21:54:23.073 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5500 IOs, 2458844160 bytes 2026-02-22T21:54:23.126 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5600 IOs, 2501040128 bytes 2026-02-22T21:54:23.195 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5700 IOs, 2566409216 bytes 2026-02-22T21:54:23.258 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5800 IOs, 2643513344 bytes 2026-02-22T21:54:23.322 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5900 IOs, 2702615552 bytes 2026-02-22T21:54:23.359 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6000 IOs, 2726777856 bytes 2026-02-22T21:54:23.409 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6100 IOs, 2771693568 bytes 2026-02-22T21:54:23.452 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6200 IOs, 2790066176 bytes 2026-02-22T21:54:23.502 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6300 IOs, 2826927104 bytes 2026-02-22T21:54:23.547 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6400 IOs, 2860001280 bytes 2026-02-22T21:54:23.589 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6500 IOs, 2876751872 bytes 2026-02-22T21:54:23.636 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6600 IOs, 2919185408 bytes 2026-02-22T21:54:23.686 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6700 IOs, 2962350080 bytes 2026-02-22T21:54:23.762 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6800 IOs, 3039167488 bytes 2026-02-22T21:54:23.829 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6900 IOs, 3105583104 bytes 2026-02-22T21:54:23.871 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7000 IOs, 3138371584 bytes 2026-02-22T21:54:23.936 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7100 IOs, 3224858624 bytes 2026-02-22T21:54:23.993 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7200 IOs, 3261345792 bytes 2026-02-22T21:54:24.037 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7300 IOs, 3286878208 bytes 2026-02-22T21:54:24.067 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7400 IOs, 3288920064 bytes 2026-02-22T21:54:24.112 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7500 IOs, 3325186048 bytes 2026-02-22T21:54:24.170 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.5502121666526963) > weight (0.25) 2026-02-22T21:54:24.170 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.5403846001340294) > weight (0.25) 2026-02-22T21:54:24.170 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.2 2026-02-22T21:54:24.170 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Sent signal 15 2026-02-22T21:54:24.170 INFO:tasks.rbd_mirror_thrash:waiting for 5.541109012883868 secs before reviving daemons 2026-02-22T21:54:24.171 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial026.stderr:2026-02-22T21:54:24.170+0000 7fb8cd50a640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2 (PID: 48220) UID: 1000 2026-02-22T21:54:24.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7600 IOs, 3384982528 bytes 2026-02-22T21:54:24.218 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7700 IOs, 3385590784 bytes 2026-02-22T21:54:24.281 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7800 IOs, 3477915648 bytes 2026-02-22T21:54:24.325 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7900 IOs, 3479592960 bytes 2026-02-22T21:54:24.375 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8000 IOs, 3522900992 bytes 2026-02-22T21:54:24.419 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8100 IOs, 3561212928 bytes 2026-02-22T21:54:24.451 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8200 IOs, 3568007168 bytes 2026-02-22T21:54:24.528 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8300 IOs, 3640025088 bytes 2026-02-22T21:54:24.557 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8400 IOs, 3652900864 bytes 2026-02-22T21:54:24.599 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8500 IOs, 3679549440 bytes 2026-02-22T21:54:24.638 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8600 IOs, 3700283392 bytes 2026-02-22T21:54:24.688 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8700 IOs, 3737978880 bytes 2026-02-22T21:54:24.731 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8800 IOs, 3774977024 bytes 2026-02-22T21:54:24.794 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8900 IOs, 3855997952 bytes 2026-02-22T21:54:24.864 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9000 IOs, 3924275200 bytes 2026-02-22T21:54:24.896 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9100 IOs, 3945272320 bytes 2026-02-22T21:54:24.955 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9200 IOs, 3981673472 bytes 2026-02-22T21:54:25.010 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9300 IOs, 4042762240 bytes 2026-02-22T21:54:25.042 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9400 IOs, 4063898624 bytes 2026-02-22T21:54:25.096 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9500 IOs, 4097170432 bytes 2026-02-22T21:54:25.141 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9600 IOs, 4124610560 bytes 2026-02-22T21:54:25.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9700 IOs, 4148830208 bytes 2026-02-22T21:54:25.257 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9800 IOs, 4200043520 bytes 2026-02-22T21:54:25.292 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9900 IOs, 4222446592 bytes 2026-02-22T21:54:25.346 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10000 IOs, 4267981824 bytes 2026-02-22T21:54:25.392 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10100 IOs, 4309969920 bytes 2026-02-22T21:54:25.444 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10200 IOs, 4342887424 bytes 2026-02-22T21:54:25.507 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10300 IOs, 4424664064 bytes 2026-02-22T21:54:25.586 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10400 IOs, 4495936512 bytes 2026-02-22T21:54:25.645 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10500 IOs, 4545491968 bytes 2026-02-22T21:54:25.710 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10600 IOs, 4624828416 bytes 2026-02-22T21:54:25.757 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10700 IOs, 4651589632 bytes 2026-02-22T21:54:25.785 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10800 IOs, 4655803392 bytes 2026-02-22T21:54:25.814 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10900 IOs, 4656412672 bytes 2026-02-22T21:54:25.850 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11000 IOs, 4677132288 bytes 2026-02-22T21:54:25.895 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11100 IOs, 4722105344 bytes 2026-02-22T21:54:25.945 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11200 IOs, 4764420096 bytes 2026-02-22T21:54:26.015 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11300 IOs, 4850632704 bytes 2026-02-22T21:54:26.073 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11400 IOs, 4892559360 bytes 2026-02-22T21:54:26.145 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11500 IOs, 4982385664 bytes 2026-02-22T21:54:26.220 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11600 IOs, 5041349632 bytes 2026-02-22T21:54:26.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11700 IOs, 5114588160 bytes 2026-02-22T21:54:26.344 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11800 IOs, 5179841536 bytes 2026-02-22T21:54:26.402 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11900 IOs, 5244071936 bytes 2026-02-22T21:54:26.452 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12000 IOs, 5283579904 bytes 2026-02-22T21:54:26.511 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12100 IOs, 5360092160 bytes 2026-02-22T21:54:26.570 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12200 IOs, 5384242176 bytes 2026-02-22T21:54:26.612 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12300 IOs, 5414729728 bytes 2026-02-22T21:54:26.673 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12400 IOs, 5478991872 bytes 2026-02-22T21:54:26.755 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12500 IOs, 5603491840 bytes 2026-02-22T21:54:26.808 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12600 IOs, 5633215488 bytes 2026-02-22T21:54:26.857 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12700 IOs, 5693374464 bytes 2026-02-22T21:54:26.925 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12800 IOs, 5764303872 bytes 2026-02-22T21:54:26.980 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12900 IOs, 5815531520 bytes 2026-02-22T21:54:27.008 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13000 IOs, 5816133632 bytes 2026-02-22T21:54:27.049 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13100 IOs, 5839819776 bytes 2026-02-22T21:54:27.175 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ error_code=124 2026-02-22T21:54:27.175 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set -e 2026-02-22T21:54:27.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' 124 -eq 124 ']' 2026-02-22T21:54:27.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:54:27.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T21:54:27.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_40 2026-02-22T21:54:27.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image_and_enable_mirror cluster2 mirror image_40 snapshot 128M 2026-02-22T21:54:27.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:54:27.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:54:27.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:54:27.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:54:27.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_40 2026-02-22T21:54:27.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:54:27.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T21:54:27.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n snapshot ']' 2026-02-22T21:54:27.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:54:27.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image cluster2 mirror image_40 128M 2026-02-22T21:54:27.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:54:27.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:54:27.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:54:27.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:54:27.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_40 2026-02-22T21:54:27.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:54:27.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local size=128 2026-02-22T21:54:27.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n 128M ']' 2026-02-22T21:54:27.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ size=128M 2026-02-22T21:54:27.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:54:27.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_40 2026-02-22T21:54:27.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T21:54:27.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:54:27.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' image = image ']' 2026-02-22T21:54:27.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:54:27.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ enable_mirror cluster2 mirror image_40 snapshot 2026-02-22T21:54:27.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:54:27.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:54:27.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_40 2026-02-22T21:54:27.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T21:54:27.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_40 snapshot 2026-02-22T21:54:28.145 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Mirroring enabled 2026-02-22T21:54:28.149 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 info mirror/image_40 2026-02-22T21:54:28.376 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_40': 2026-02-22T21:54:28.376 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 128 MiB in 32 objects 2026-02-22T21:54:28.376 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T21:54:28.376 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 1 2026-02-22T21:54:28.376 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 142b6bdef874 2026-02-22T21:54:28.376 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.142b6bdef874 2026-02-22T21:54:28.376 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T21:54:28.376 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock 2026-02-22T21:54:28.377 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T21:54:28.377 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T21:54:28.377 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:54:27 2026 2026-02-22T21:54:28.377 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:54:27 2026 2026-02-22T21:54:28.377 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:54:27 2026 2026-02-22T21:54:28.377 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T21:54:28.377 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T21:54:28.377 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: c57ae3a7-f4fd-4810-a324-ac5f588ec54e 2026-02-22T21:54:28.377 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: true 2026-02-22T21:54:28.379 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n '' ']' 2026-02-22T21:54:28.379 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ stress_write_image cluster2 mirror image_40 2026-02-22T21:54:28.379 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:54:28.379 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:54:28.379 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_40 2026-02-22T21:54:28.379 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-22T21:54:28.380 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local duration=9 2026-02-22T21:54:28.381 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set +e 2026-02-22T21:54:28.381 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ timeout 9s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_40 --debug-rbd=20 --debug-journaler=20 2026-02-22T21:54:28.460 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:100 IOs, 30787584 bytes 2026-02-22T21:54:28.498 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:200 IOs, 86558720 bytes 2026-02-22T21:54:28.569 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:300 IOs, 141699072 bytes 2026-02-22T21:54:28.635 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:400 IOs, 211484672 bytes 2026-02-22T21:54:28.669 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:500 IOs, 227408896 bytes 2026-02-22T21:54:28.719 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:600 IOs, 258814976 bytes 2026-02-22T21:54:28.751 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:700 IOs, 266563584 bytes 2026-02-22T21:54:28.794 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:800 IOs, 306878464 bytes 2026-02-22T21:54:28.888 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:900 IOs, 421803008 bytes 2026-02-22T21:54:28.947 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1000 IOs, 465758208 bytes 2026-02-22T21:54:28.975 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1100 IOs, 466592768 bytes 2026-02-22T21:54:29.022 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1200 IOs, 500249600 bytes 2026-02-22T21:54:29.069 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1300 IOs, 529057792 bytes 2026-02-22T21:54:29.109 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1400 IOs, 551237632 bytes 2026-02-22T21:54:29.173 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1500 IOs, 615249920 bytes 2026-02-22T21:54:29.214 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1600 IOs, 625123328 bytes 2026-02-22T21:54:29.286 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1700 IOs, 688804864 bytes 2026-02-22T21:54:29.337 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1800 IOs, 740709376 bytes 2026-02-22T21:54:29.408 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1900 IOs, 804909056 bytes 2026-02-22T21:54:29.484 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2000 IOs, 881071104 bytes 2026-02-22T21:54:29.524 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2100 IOs, 891463680 bytes 2026-02-22T21:54:29.558 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2200 IOs, 900316160 bytes 2026-02-22T21:54:29.614 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2300 IOs, 939567104 bytes 2026-02-22T21:54:29.665 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2400 IOs, 981654528 bytes 2026-02-22T21:54:29.712 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.2 2026-02-22T21:54:29.712 INFO:teuthology.orchestra.run:waiting for 600 2026-02-22T21:54:29.712 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.2 2026-02-22T21:54:29.712 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Restarting daemon 2026-02-22T21:54:29.713 DEBUG:teuthology.orchestra.run.trial026:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2 2026-02-22T21:54:29.715 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Started 2026-02-22T21:54:29.716 INFO:tasks.rbd_mirror_thrash:waiting for 87.5827118867985 secs before thrashing 2026-02-22T21:54:29.727 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2500 IOs, 1044981760 bytes 2026-02-22T21:54:29.787 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2600 IOs, 1085198336 bytes 2026-02-22T21:54:29.848 INFO:tasks.ceph.cluster2.osd.2.trial026.stderr:2026-02-22T21:54:29.847+0000 7f32e29d2640 -1 reset not still connected to 0x5576a9724d00 2026-02-22T21:54:29.851 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2700 IOs, 1135568896 bytes 2026-02-22T21:54:29.892 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2800 IOs, 1153666048 bytes 2026-02-22T21:54:29.947 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2900 IOs, 1225391104 bytes 2026-02-22T21:54:30.025 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3000 IOs, 1286251520 bytes 2026-02-22T21:54:30.077 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3100 IOs, 1344235520 bytes 2026-02-22T21:54:30.143 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3200 IOs, 1381654528 bytes 2026-02-22T21:54:30.188 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3300 IOs, 1413648384 bytes 2026-02-22T21:54:30.245 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3400 IOs, 1458849792 bytes 2026-02-22T21:54:30.290 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3500 IOs, 1478299648 bytes 2026-02-22T21:54:30.336 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3600 IOs, 1526431744 bytes 2026-02-22T21:54:30.384 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3700 IOs, 1552936960 bytes 2026-02-22T21:54:30.420 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3800 IOs, 1589612544 bytes 2026-02-22T21:54:30.483 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3900 IOs, 1637888000 bytes 2026-02-22T21:54:30.535 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4000 IOs, 1667124224 bytes 2026-02-22T21:54:30.595 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4100 IOs, 1705154560 bytes 2026-02-22T21:54:30.660 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4200 IOs, 1775191040 bytes 2026-02-22T21:54:30.697 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4300 IOs, 1792587776 bytes 2026-02-22T21:54:30.749 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4400 IOs, 1842125824 bytes 2026-02-22T21:54:30.789 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4500 IOs, 1865548800 bytes 2026-02-22T21:54:30.836 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4600 IOs, 1897613312 bytes 2026-02-22T21:54:30.877 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4700 IOs, 1923692544 bytes 2026-02-22T21:54:30.939 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4800 IOs, 1997848576 bytes 2026-02-22T21:54:31.001 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4900 IOs, 2056793088 bytes 2026-02-22T21:54:31.046 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5000 IOs, 2093768704 bytes 2026-02-22T21:54:31.082 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5100 IOs, 2107018240 bytes 2026-02-22T21:54:31.124 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5200 IOs, 2146578432 bytes 2026-02-22T21:54:31.236 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5300 IOs, 2287424512 bytes 2026-02-22T21:54:31.286 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5400 IOs, 2326081536 bytes 2026-02-22T21:54:31.349 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5500 IOs, 2398949376 bytes 2026-02-22T21:54:31.413 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5600 IOs, 2473004032 bytes 2026-02-22T21:54:31.479 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5700 IOs, 2549001216 bytes 2026-02-22T21:54:31.527 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5800 IOs, 2580987904 bytes 2026-02-22T21:54:31.571 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5900 IOs, 2602498048 bytes 2026-02-22T21:54:31.630 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6000 IOs, 2650998784 bytes 2026-02-22T21:54:31.667 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6100 IOs, 2683055104 bytes 2026-02-22T21:54:31.717 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6200 IOs, 2692426752 bytes 2026-02-22T21:54:31.749 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6300 IOs, 2698447872 bytes 2026-02-22T21:54:31.800 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6400 IOs, 2753808384 bytes 2026-02-22T21:54:31.857 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6500 IOs, 2786580480 bytes 2026-02-22T21:54:31.922 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6600 IOs, 2853688320 bytes 2026-02-22T21:54:31.994 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6700 IOs, 2926346240 bytes 2026-02-22T21:54:32.034 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6800 IOs, 2950889472 bytes 2026-02-22T21:54:32.103 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6900 IOs, 3022200832 bytes 2026-02-22T21:54:32.136 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7000 IOs, 3033327616 bytes 2026-02-22T21:54:32.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7100 IOs, 3089850368 bytes 2026-02-22T21:54:32.267 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7200 IOs, 3183152128 bytes 2026-02-22T21:54:32.330 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7300 IOs, 3223343104 bytes 2026-02-22T21:54:32.406 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7400 IOs, 3293047808 bytes 2026-02-22T21:54:32.448 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7500 IOs, 3318388736 bytes 2026-02-22T21:54:32.508 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7600 IOs, 3380134912 bytes 2026-02-22T21:54:32.582 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7700 IOs, 3457656832 bytes 2026-02-22T21:54:32.657 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7800 IOs, 3550126080 bytes 2026-02-22T21:54:32.708 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7900 IOs, 3586868224 bytes 2026-02-22T21:54:32.737 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8000 IOs, 3591238656 bytes 2026-02-22T21:54:32.782 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8100 IOs, 3633152000 bytes 2026-02-22T21:54:32.829 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8200 IOs, 3664351232 bytes 2026-02-22T21:54:32.886 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8300 IOs, 3717130240 bytes 2026-02-22T21:54:32.958 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8400 IOs, 3800381440 bytes 2026-02-22T21:54:33.014 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8500 IOs, 3860558848 bytes 2026-02-22T21:54:33.082 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8600 IOs, 3948395520 bytes 2026-02-22T21:54:33.135 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8700 IOs, 3986310144 bytes 2026-02-22T21:54:33.184 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8800 IOs, 4032140288 bytes 2026-02-22T21:54:33.247 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8900 IOs, 4097502208 bytes 2026-02-22T21:54:33.301 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9000 IOs, 4134304768 bytes 2026-02-22T21:54:33.341 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9100 IOs, 4158793728 bytes 2026-02-22T21:54:33.384 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9200 IOs, 4188892160 bytes 2026-02-22T21:54:33.431 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9300 IOs, 4233755648 bytes 2026-02-22T21:54:33.483 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9400 IOs, 4278109184 bytes 2026-02-22T21:54:33.528 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9500 IOs, 4302081024 bytes 2026-02-22T21:54:33.570 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9600 IOs, 4322530304 bytes 2026-02-22T21:54:33.614 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9700 IOs, 4351768576 bytes 2026-02-22T21:54:33.702 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9800 IOs, 4463222784 bytes 2026-02-22T21:54:33.746 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9900 IOs, 4484100096 bytes 2026-02-22T21:54:33.804 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10000 IOs, 4543026176 bytes 2026-02-22T21:54:33.865 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10100 IOs, 4624481280 bytes 2026-02-22T21:54:33.909 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10200 IOs, 4643492864 bytes 2026-02-22T21:54:33.952 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10300 IOs, 4673034240 bytes 2026-02-22T21:54:34.012 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10400 IOs, 4724199424 bytes 2026-02-22T21:54:34.051 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10500 IOs, 4745374720 bytes 2026-02-22T21:54:34.084 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10600 IOs, 4755062784 bytes 2026-02-22T21:54:34.148 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10700 IOs, 4822379520 bytes 2026-02-22T21:54:34.188 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10800 IOs, 4843656192 bytes 2026-02-22T21:54:34.239 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10900 IOs, 4889815040 bytes 2026-02-22T21:54:34.274 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11000 IOs, 4895243264 bytes 2026-02-22T21:54:34.317 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11100 IOs, 4937759744 bytes 2026-02-22T21:54:34.358 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11200 IOs, 4944574464 bytes 2026-02-22T21:54:34.413 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11300 IOs, 5006458880 bytes 2026-02-22T21:54:34.514 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11400 IOs, 5138593792 bytes 2026-02-22T21:54:34.575 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11500 IOs, 5187410944 bytes 2026-02-22T21:54:34.627 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11600 IOs, 5214225408 bytes 2026-02-22T21:54:34.668 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11700 IOs, 5233562624 bytes 2026-02-22T21:54:34.708 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11800 IOs, 5254519808 bytes 2026-02-22T21:54:34.755 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11900 IOs, 5282180096 bytes 2026-02-22T21:54:34.793 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12000 IOs, 5301544960 bytes 2026-02-22T21:54:34.821 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12100 IOs, 5304067072 bytes 2026-02-22T21:54:34.924 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12200 IOs, 5419757568 bytes 2026-02-22T21:54:34.969 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12300 IOs, 5447128064 bytes 2026-02-22T21:54:35.008 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12400 IOs, 5472074752 bytes 2026-02-22T21:54:35.042 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12500 IOs, 5490889728 bytes 2026-02-22T21:54:35.105 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12600 IOs, 5560554496 bytes 2026-02-22T21:54:35.173 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12700 IOs, 5620164608 bytes 2026-02-22T21:54:35.248 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12800 IOs, 5687934976 bytes 2026-02-22T21:54:35.298 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12900 IOs, 5734925312 bytes 2026-02-22T21:54:35.369 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13000 IOs, 5783530496 bytes 2026-02-22T21:54:35.431 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13100 IOs, 5849781248 bytes 2026-02-22T21:54:35.458 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13200 IOs, 5850390528 bytes 2026-02-22T21:54:35.490 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13300 IOs, 5869522944 bytes 2026-02-22T21:54:35.526 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13400 IOs, 5876690944 bytes 2026-02-22T21:54:35.587 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13500 IOs, 5945862144 bytes 2026-02-22T21:54:35.623 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13600 IOs, 5959643136 bytes 2026-02-22T21:54:35.670 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13700 IOs, 5995879424 bytes 2026-02-22T21:54:35.742 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13800 IOs, 6068216832 bytes 2026-02-22T21:54:35.787 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13900 IOs, 6100534272 bytes 2026-02-22T21:54:35.815 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14000 IOs, 6105185280 bytes 2026-02-22T21:54:35.871 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14100 IOs, 6167529472 bytes 2026-02-22T21:54:35.932 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14200 IOs, 6234743808 bytes 2026-02-22T21:54:35.966 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14300 IOs, 6243631104 bytes 2026-02-22T21:54:36.020 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14400 IOs, 6280057856 bytes 2026-02-22T21:54:36.102 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14500 IOs, 6396847104 bytes 2026-02-22T21:54:36.151 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14600 IOs, 6422448128 bytes 2026-02-22T21:54:36.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14700 IOs, 6451233792 bytes 2026-02-22T21:54:36.241 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14800 IOs, 6487550976 bytes 2026-02-22T21:54:36.297 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14900 IOs, 6538919936 bytes 2026-02-22T21:54:36.353 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15000 IOs, 6581758976 bytes 2026-02-22T21:54:36.413 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15100 IOs, 6645649408 bytes 2026-02-22T21:54:36.479 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15200 IOs, 6716283904 bytes 2026-02-22T21:54:36.533 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15300 IOs, 6762336256 bytes 2026-02-22T21:54:36.571 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15400 IOs, 6782336000 bytes 2026-02-22T21:54:36.607 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15500 IOs, 6803589120 bytes 2026-02-22T21:54:36.669 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15600 IOs, 6871666688 bytes 2026-02-22T21:54:36.717 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15700 IOs, 6899707904 bytes 2026-02-22T21:54:36.798 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15800 IOs, 7000182784 bytes 2026-02-22T21:54:36.881 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15900 IOs, 7098263552 bytes 2026-02-22T21:54:36.926 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16000 IOs, 7135240192 bytes 2026-02-22T21:54:36.974 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16100 IOs, 7174395904 bytes 2026-02-22T21:54:37.027 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16200 IOs, 7228959744 bytes 2026-02-22T21:54:37.064 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16300 IOs, 7252302848 bytes 2026-02-22T21:54:37.110 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16400 IOs, 7282260992 bytes 2026-02-22T21:54:37.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16500 IOs, 7339414528 bytes 2026-02-22T21:54:37.240 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16600 IOs, 7426094080 bytes 2026-02-22T21:54:37.280 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16700 IOs, 7438125056 bytes 2026-02-22T21:54:37.344 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16800 IOs, 7499743232 bytes 2026-02-22T21:54:37.386 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ error_code=124 2026-02-22T21:54:37.386 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set -e 2026-02-22T21:54:37.386 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' 124 -eq 124 ']' 2026-02-22T21:54:37.387 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:54:37.387 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T21:54:37.387 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_41 2026-02-22T21:54:37.387 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image_and_enable_mirror cluster2 mirror image_41 snapshot 128M 2026-02-22T21:54:37.387 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:54:37.387 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:54:37.387 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:54:37.387 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:54:37.387 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_41 2026-02-22T21:54:37.387 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:54:37.387 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T21:54:37.388 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n snapshot ']' 2026-02-22T21:54:37.388 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:54:37.388 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image cluster2 mirror image_41 128M 2026-02-22T21:54:37.388 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:54:37.388 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:54:37.388 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:54:37.388 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:54:37.388 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_41 2026-02-22T21:54:37.388 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:54:37.388 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local size=128 2026-02-22T21:54:37.388 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n 128M ']' 2026-02-22T21:54:37.388 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ size=128M 2026-02-22T21:54:37.388 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:54:37.389 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_41 2026-02-22T21:54:37.417 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T21:54:37.417 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:54:37.417 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' image = image ']' 2026-02-22T21:54:37.417 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:54:37.417 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ enable_mirror cluster2 mirror image_41 snapshot 2026-02-22T21:54:37.417 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:54:37.417 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:54:37.417 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_41 2026-02-22T21:54:37.417 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T21:54:37.417 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_41 snapshot 2026-02-22T21:54:38.154 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Mirroring enabled 2026-02-22T21:54:38.158 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 info mirror/image_41 2026-02-22T21:54:38.183 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_41': 2026-02-22T21:54:38.183 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 128 MiB in 32 objects 2026-02-22T21:54:38.183 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T21:54:38.183 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 1 2026-02-22T21:54:38.183 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 1440125fd956 2026-02-22T21:54:38.183 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.1440125fd956 2026-02-22T21:54:38.183 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T21:54:38.184 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock 2026-02-22T21:54:38.184 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T21:54:38.184 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T21:54:38.184 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:54:37 2026 2026-02-22T21:54:38.184 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:54:37 2026 2026-02-22T21:54:38.184 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:54:37 2026 2026-02-22T21:54:38.184 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T21:54:38.184 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T21:54:38.184 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: 69209f84-531b-4558-bff7-497f660589ca 2026-02-22T21:54:38.184 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: true 2026-02-22T21:54:38.185 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n '' ']' 2026-02-22T21:54:38.185 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ stress_write_image cluster2 mirror image_41 2026-02-22T21:54:38.185 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:54:38.185 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:54:38.186 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_41 2026-02-22T21:54:38.186 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-22T21:54:38.186 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local duration=6 2026-02-22T21:54:38.186 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set +e 2026-02-22T21:54:38.187 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ timeout 6s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_41 --debug-rbd=20 --debug-journaler=20 2026-02-22T21:54:38.261 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:100 IOs, 25652224 bytes 2026-02-22T21:54:38.300 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:200 IOs, 49826816 bytes 2026-02-22T21:54:38.340 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:300 IOs, 77038592 bytes 2026-02-22T21:54:38.382 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:400 IOs, 101407744 bytes 2026-02-22T21:54:38.416 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:500 IOs, 120416256 bytes 2026-02-22T21:54:38.465 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:600 IOs, 158200832 bytes 2026-02-22T21:54:38.502 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:700 IOs, 171709440 bytes 2026-02-22T21:54:38.557 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:800 IOs, 213693440 bytes 2026-02-22T21:54:38.604 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:900 IOs, 236637184 bytes 2026-02-22T21:54:38.636 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1000 IOs, 243638272 bytes 2026-02-22T21:54:38.670 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1100 IOs, 256225280 bytes 2026-02-22T21:54:38.706 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1200 IOs, 268201984 bytes 2026-02-22T21:54:38.761 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1300 IOs, 319764480 bytes 2026-02-22T21:54:38.801 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1400 IOs, 348921856 bytes 2026-02-22T21:54:38.829 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1500 IOs, 349535232 bytes 2026-02-22T21:54:38.891 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1600 IOs, 428304384 bytes 2026-02-22T21:54:38.942 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1700 IOs, 458238976 bytes 2026-02-22T21:54:38.983 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1800 IOs, 488930304 bytes 2026-02-22T21:54:39.044 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1900 IOs, 532287488 bytes 2026-02-22T21:54:39.091 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2000 IOs, 574284800 bytes 2026-02-22T21:54:39.142 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2100 IOs, 604908544 bytes 2026-02-22T21:54:39.170 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2200 IOs, 605519872 bytes 2026-02-22T21:54:39.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2300 IOs, 658675712 bytes 2026-02-22T21:54:39.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2400 IOs, 711782400 bytes 2026-02-22T21:54:39.327 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2500 IOs, 751233024 bytes 2026-02-22T21:54:39.389 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2600 IOs, 807644160 bytes 2026-02-22T21:54:39.452 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2700 IOs, 882325504 bytes 2026-02-22T21:54:39.527 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2800 IOs, 984332288 bytes 2026-02-22T21:54:39.583 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2900 IOs, 1038093312 bytes 2026-02-22T21:54:39.644 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3000 IOs, 1095395328 bytes 2026-02-22T21:54:39.697 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3100 IOs, 1130933248 bytes 2026-02-22T21:54:39.733 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3200 IOs, 1141910528 bytes 2026-02-22T21:54:39.803 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3300 IOs, 1222605824 bytes 2026-02-22T21:54:39.853 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3400 IOs, 1258575872 bytes 2026-02-22T21:54:39.935 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3500 IOs, 1354978304 bytes 2026-02-22T21:54:39.972 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3600 IOs, 1359879168 bytes 2026-02-22T21:54:40.069 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3700 IOs, 1513606144 bytes 2026-02-22T21:54:40.107 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3800 IOs, 1561936896 bytes 2026-02-22T21:54:40.155 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3900 IOs, 1575141376 bytes 2026-02-22T21:54:40.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4000 IOs, 1625485312 bytes 2026-02-22T21:54:40.263 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4100 IOs, 1670683648 bytes 2026-02-22T21:54:40.314 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4200 IOs, 1713345536 bytes 2026-02-22T21:54:40.343 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4300 IOs, 1715958784 bytes 2026-02-22T21:54:40.368 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4400 IOs, 1716576256 bytes 2026-02-22T21:54:40.431 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4500 IOs, 1779823616 bytes 2026-02-22T21:54:40.519 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4600 IOs, 1876955136 bytes 2026-02-22T21:54:40.598 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4700 IOs, 1962371072 bytes 2026-02-22T21:54:40.667 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4800 IOs, 2027887616 bytes 2026-02-22T21:54:40.740 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4900 IOs, 2112833536 bytes 2026-02-22T21:54:40.776 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5000 IOs, 2113683456 bytes 2026-02-22T21:54:40.842 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5100 IOs, 2196405248 bytes 2026-02-22T21:54:40.915 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5200 IOs, 2252174336 bytes 2026-02-22T21:54:41.007 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5300 IOs, 2369259520 bytes 2026-02-22T21:54:41.055 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5400 IOs, 2396695552 bytes 2026-02-22T21:54:41.110 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5500 IOs, 2458703872 bytes 2026-02-22T21:54:41.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5600 IOs, 2511517696 bytes 2026-02-22T21:54:41.220 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5700 IOs, 2544510976 bytes 2026-02-22T21:54:41.250 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5800 IOs, 2552671232 bytes 2026-02-22T21:54:41.299 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5900 IOs, 2596467712 bytes 2026-02-22T21:54:41.356 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6000 IOs, 2637783040 bytes 2026-02-22T21:54:41.407 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6100 IOs, 2686119936 bytes 2026-02-22T21:54:41.462 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6200 IOs, 2721907712 bytes 2026-02-22T21:54:41.491 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6300 IOs, 2728628224 bytes 2026-02-22T21:54:41.536 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6400 IOs, 2760451072 bytes 2026-02-22T21:54:41.593 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6500 IOs, 2810298368 bytes 2026-02-22T21:54:41.651 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6600 IOs, 2860048384 bytes 2026-02-22T21:54:41.703 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6700 IOs, 2886569984 bytes 2026-02-22T21:54:41.762 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6800 IOs, 2935402496 bytes 2026-02-22T21:54:41.807 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6900 IOs, 2973431808 bytes 2026-02-22T21:54:41.840 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7000 IOs, 2993267712 bytes 2026-02-22T21:54:41.924 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7100 IOs, 3059160064 bytes 2026-02-22T21:54:41.970 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7200 IOs, 3089708032 bytes 2026-02-22T21:54:41.997 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7300 IOs, 3090318336 bytes 2026-02-22T21:54:42.033 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7400 IOs, 3105128448 bytes 2026-02-22T21:54:42.090 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7500 IOs, 3155207168 bytes 2026-02-22T21:54:42.119 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7600 IOs, 3179490304 bytes 2026-02-22T21:54:42.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7700 IOs, 3251853312 bytes 2026-02-22T21:54:42.248 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7800 IOs, 3298152448 bytes 2026-02-22T21:54:42.308 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7900 IOs, 3353885696 bytes 2026-02-22T21:54:42.351 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8000 IOs, 3383154688 bytes 2026-02-22T21:54:42.417 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8100 IOs, 3442200576 bytes 2026-02-22T21:54:42.456 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8200 IOs, 3474130944 bytes 2026-02-22T21:54:42.502 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8300 IOs, 3492407296 bytes 2026-02-22T21:54:42.545 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8400 IOs, 3520743424 bytes 2026-02-22T21:54:42.599 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8500 IOs, 3577315328 bytes 2026-02-22T21:54:42.666 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8600 IOs, 3627508736 bytes 2026-02-22T21:54:42.747 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8700 IOs, 3738468352 bytes 2026-02-22T21:54:42.808 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8800 IOs, 3777888256 bytes 2026-02-22T21:54:42.867 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8900 IOs, 3830018048 bytes 2026-02-22T21:54:42.923 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9000 IOs, 3874835456 bytes 2026-02-22T21:54:42.960 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9100 IOs, 3898470400 bytes 2026-02-22T21:54:43.004 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9200 IOs, 3932673024 bytes 2026-02-22T21:54:43.040 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9300 IOs, 3933277184 bytes 2026-02-22T21:54:43.098 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9400 IOs, 3994186752 bytes 2026-02-22T21:54:43.137 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9500 IOs, 4034383872 bytes 2026-02-22T21:54:43.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9600 IOs, 4068669440 bytes 2026-02-22T21:54:43.265 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9700 IOs, 4150032384 bytes 2026-02-22T21:54:43.302 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9800 IOs, 4171623424 bytes 2026-02-22T21:54:43.356 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9900 IOs, 4202850304 bytes 2026-02-22T21:54:43.421 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10000 IOs, 4266561536 bytes 2026-02-22T21:54:43.480 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10100 IOs, 4316275712 bytes 2026-02-22T21:54:43.551 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10200 IOs, 4396718080 bytes 2026-02-22T21:54:43.589 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10300 IOs, 4409465856 bytes 2026-02-22T21:54:43.617 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10400 IOs, 4419315712 bytes 2026-02-22T21:54:43.678 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10500 IOs, 4469380096 bytes 2026-02-22T21:54:43.762 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10600 IOs, 4555998208 bytes 2026-02-22T21:54:43.811 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10700 IOs, 4597127168 bytes 2026-02-22T21:54:43.860 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10800 IOs, 4639579136 bytes 2026-02-22T21:54:43.912 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10900 IOs, 4695369728 bytes 2026-02-22T21:54:43.969 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11000 IOs, 4736663552 bytes 2026-02-22T21:54:43.973 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.6952829053509821) > weight (0.25) 2026-02-22T21:54:43.973 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.9746393310006066) > weight (0.25) 2026-02-22T21:54:43.973 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.2 2026-02-22T21:54:43.974 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Sent signal 15 2026-02-22T21:54:43.974 INFO:tasks.rbd_mirror_thrash:waiting for 4.933458512664823 secs before reviving daemons 2026-02-22T21:54:43.974 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial026.stderr:2026-02-22T21:54:43.973+0000 7ff6f4499640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.2 (PID: 57432) UID: 1000 2026-02-22T21:54:44.021 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11100 IOs, 4776500224 bytes 2026-02-22T21:54:44.081 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11200 IOs, 4856153088 bytes 2026-02-22T21:54:44.149 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11300 IOs, 4910229504 bytes 2026-02-22T21:54:44.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11400 IOs, 4917665792 bytes 2026-02-22T21:54:44.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ error_code=124 2026-02-22T21:54:44.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set -e 2026-02-22T21:54:44.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' 124 -eq 124 ']' 2026-02-22T21:54:44.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:54:44.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T21:54:44.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_42 2026-02-22T21:54:44.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image_and_enable_mirror cluster2 mirror image_42 snapshot 128M 2026-02-22T21:54:44.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:54:44.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:54:44.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:54:44.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:54:44.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_42 2026-02-22T21:54:44.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:54:44.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T21:54:44.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n snapshot ']' 2026-02-22T21:54:44.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:54:44.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image cluster2 mirror image_42 128M 2026-02-22T21:54:44.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:54:44.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:54:44.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:54:44.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:54:44.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_42 2026-02-22T21:54:44.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:54:44.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local size=128 2026-02-22T21:54:44.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n 128M ']' 2026-02-22T21:54:44.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ size=128M 2026-02-22T21:54:44.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:54:44.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_42 2026-02-22T21:54:44.223 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T21:54:44.223 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:54:44.223 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' image = image ']' 2026-02-22T21:54:44.223 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:54:44.223 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ enable_mirror cluster2 mirror image_42 snapshot 2026-02-22T21:54:44.223 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:54:44.223 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:54:44.223 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_42 2026-02-22T21:54:44.223 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T21:54:44.223 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_42 snapshot 2026-02-22T21:54:45.163 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Mirroring enabled 2026-02-22T21:54:45.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 info mirror/image_42 2026-02-22T21:54:45.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_42': 2026-02-22T21:54:45.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 128 MiB in 32 objects 2026-02-22T21:54:45.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T21:54:45.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 1 2026-02-22T21:54:45.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 144cec7d0319 2026-02-22T21:54:45.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.144cec7d0319 2026-02-22T21:54:45.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T21:54:45.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock 2026-02-22T21:54:45.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T21:54:45.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T21:54:45.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:54:44 2026 2026-02-22T21:54:45.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:54:44 2026 2026-02-22T21:54:45.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:54:44 2026 2026-02-22T21:54:45.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T21:54:45.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T21:54:45.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: 75d1530f-c13b-4f15-aac1-c10cd70d3738 2026-02-22T21:54:45.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: true 2026-02-22T21:54:45.195 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n '' ']' 2026-02-22T21:54:45.196 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ stress_write_image cluster2 mirror image_42 2026-02-22T21:54:45.196 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:54:45.196 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:54:45.196 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_42 2026-02-22T21:54:45.196 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-22T21:54:45.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local duration=9 2026-02-22T21:54:45.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set +e 2026-02-22T21:54:45.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ timeout 9s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_42 --debug-rbd=20 --debug-journaler=20 2026-02-22T21:54:45.281 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:100 IOs, 60461056 bytes 2026-02-22T21:54:45.339 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:200 IOs, 90355712 bytes 2026-02-22T21:54:45.391 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:300 IOs, 135629824 bytes 2026-02-22T21:54:45.429 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:400 IOs, 183578624 bytes 2026-02-22T21:54:45.507 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:500 IOs, 240234496 bytes 2026-02-22T21:54:45.535 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:600 IOs, 240855040 bytes 2026-02-22T21:54:45.599 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:700 IOs, 299670528 bytes 2026-02-22T21:54:45.627 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:800 IOs, 300266496 bytes 2026-02-22T21:54:45.685 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:900 IOs, 357316608 bytes 2026-02-22T21:54:45.726 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1000 IOs, 399470592 bytes 2026-02-22T21:54:45.768 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1100 IOs, 407944192 bytes 2026-02-22T21:54:45.821 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1200 IOs, 450266112 bytes 2026-02-22T21:54:45.849 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1300 IOs, 454929408 bytes 2026-02-22T21:54:45.890 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1400 IOs, 475627520 bytes 2026-02-22T21:54:45.951 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1500 IOs, 538450944 bytes 2026-02-22T21:54:45.986 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1600 IOs, 549689344 bytes 2026-02-22T21:54:46.027 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1700 IOs, 577430528 bytes 2026-02-22T21:54:46.066 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1800 IOs, 600259584 bytes 2026-02-22T21:54:46.117 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1900 IOs, 649833472 bytes 2026-02-22T21:54:46.161 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2000 IOs, 672882688 bytes 2026-02-22T21:54:46.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2100 IOs, 700109824 bytes 2026-02-22T21:54:46.256 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2200 IOs, 750332928 bytes 2026-02-22T21:54:46.293 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2300 IOs, 778242048 bytes 2026-02-22T21:54:46.376 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2400 IOs, 852041728 bytes 2026-02-22T21:54:46.444 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2500 IOs, 929804288 bytes 2026-02-22T21:54:46.501 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2600 IOs, 985445376 bytes 2026-02-22T21:54:46.540 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2700 IOs, 1009161216 bytes 2026-02-22T21:54:46.620 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2800 IOs, 1115909120 bytes 2026-02-22T21:54:46.660 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2900 IOs, 1132867584 bytes 2026-02-22T21:54:46.698 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3000 IOs, 1148268544 bytes 2026-02-22T21:54:46.736 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3100 IOs, 1161838592 bytes 2026-02-22T21:54:46.764 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3200 IOs, 1164139520 bytes 2026-02-22T21:54:46.794 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3300 IOs, 1178823680 bytes 2026-02-22T21:54:46.853 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3400 IOs, 1237175296 bytes 2026-02-22T21:54:46.934 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3500 IOs, 1310613504 bytes 2026-02-22T21:54:46.961 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3600 IOs, 1311221760 bytes 2026-02-22T21:54:47.003 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3700 IOs, 1338737664 bytes 2026-02-22T21:54:47.059 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3800 IOs, 1389014016 bytes 2026-02-22T21:54:47.134 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3900 IOs, 1481071616 bytes 2026-02-22T21:54:47.171 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4000 IOs, 1495351296 bytes 2026-02-22T21:54:47.219 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4100 IOs, 1526387712 bytes 2026-02-22T21:54:47.259 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4200 IOs, 1551965184 bytes 2026-02-22T21:54:47.325 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4300 IOs, 1623471104 bytes 2026-02-22T21:54:47.363 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4400 IOs, 1624087552 bytes 2026-02-22T21:54:47.420 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4500 IOs, 1657160704 bytes 2026-02-22T21:54:47.477 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4600 IOs, 1704689664 bytes 2026-02-22T21:54:47.514 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4700 IOs, 1722820608 bytes 2026-02-22T21:54:47.565 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4800 IOs, 1763988480 bytes 2026-02-22T21:54:47.621 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4900 IOs, 1820420096 bytes 2026-02-22T21:54:47.687 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5000 IOs, 1892559872 bytes 2026-02-22T21:54:47.756 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5100 IOs, 1971117056 bytes 2026-02-22T21:54:47.800 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5200 IOs, 1980785664 bytes 2026-02-22T21:54:47.832 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5300 IOs, 1996423168 bytes 2026-02-22T21:54:47.901 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5400 IOs, 2070321152 bytes 2026-02-22T21:54:47.950 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5500 IOs, 2112352256 bytes 2026-02-22T21:54:47.992 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5600 IOs, 2150342656 bytes 2026-02-22T21:54:48.057 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5700 IOs, 2214974464 bytes 2026-02-22T21:54:48.088 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5800 IOs, 2229390336 bytes 2026-02-22T21:54:48.146 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5900 IOs, 2278292480 bytes 2026-02-22T21:54:48.185 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6000 IOs, 2300420096 bytes 2026-02-22T21:54:48.228 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6100 IOs, 2334553088 bytes 2026-02-22T21:54:48.265 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6200 IOs, 2359685120 bytes 2026-02-22T21:54:48.316 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6300 IOs, 2402659328 bytes 2026-02-22T21:54:48.372 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6400 IOs, 2443148288 bytes 2026-02-22T21:54:48.411 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6500 IOs, 2452477952 bytes 2026-02-22T21:54:48.471 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6600 IOs, 2504222720 bytes 2026-02-22T21:54:48.530 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6700 IOs, 2569253888 bytes 2026-02-22T21:54:48.574 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6800 IOs, 2601939968 bytes 2026-02-22T21:54:48.614 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6900 IOs, 2652086272 bytes 2026-02-22T21:54:48.853 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7000 IOs, 2659857408 bytes 2026-02-22T21:54:48.908 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.2 2026-02-22T21:54:48.908 INFO:teuthology.orchestra.run:waiting for 600 2026-02-22T21:54:48.908 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.2 2026-02-22T21:54:48.909 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Restarting daemon 2026-02-22T21:54:48.909 DEBUG:teuthology.orchestra.run.trial026:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.2 2026-02-22T21:54:48.912 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Started 2026-02-22T21:54:48.912 INFO:tasks.rbd_mirror_thrash:waiting for 102.1369631331598 secs before thrashing 2026-02-22T21:54:48.914 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7100 IOs, 2731995136 bytes 2026-02-22T21:54:48.975 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7200 IOs, 2768254976 bytes 2026-02-22T21:54:49.036 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7300 IOs, 2822126592 bytes 2026-02-22T21:54:49.100 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7400 IOs, 2880296960 bytes 2026-02-22T21:54:49.144 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7500 IOs, 2930798592 bytes 2026-02-22T21:54:49.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7600 IOs, 2964660224 bytes 2026-02-22T21:54:49.294 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7700 IOs, 3078666240 bytes 2026-02-22T21:54:49.353 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7800 IOs, 3136081920 bytes 2026-02-22T21:54:49.404 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7900 IOs, 3176933376 bytes 2026-02-22T21:54:49.448 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8000 IOs, 3207073792 bytes 2026-02-22T21:54:49.494 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8100 IOs, 3248028672 bytes 2026-02-22T21:54:49.544 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8200 IOs, 3299806208 bytes 2026-02-22T21:54:49.598 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8300 IOs, 3351173120 bytes 2026-02-22T21:54:49.631 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8400 IOs, 3357595648 bytes 2026-02-22T21:54:49.690 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8500 IOs, 3434962944 bytes 2026-02-22T21:54:49.738 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8600 IOs, 3455370240 bytes 2026-02-22T21:54:49.790 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8700 IOs, 3488823296 bytes 2026-02-22T21:54:49.835 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8800 IOs, 3533966336 bytes 2026-02-22T21:54:49.891 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8900 IOs, 3566384128 bytes 2026-02-22T21:54:49.966 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9000 IOs, 3654012928 bytes 2026-02-22T21:54:50.013 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9100 IOs, 3670810624 bytes 2026-02-22T21:54:50.049 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9200 IOs, 3688130560 bytes 2026-02-22T21:54:50.076 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9300 IOs, 3688870912 bytes 2026-02-22T21:54:50.105 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9400 IOs, 3694703616 bytes 2026-02-22T21:54:50.170 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9500 IOs, 3760800768 bytes 2026-02-22T21:54:50.228 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9600 IOs, 3813914624 bytes 2026-02-22T21:54:50.276 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9700 IOs, 3850002432 bytes 2026-02-22T21:54:50.329 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9800 IOs, 3896995840 bytes 2026-02-22T21:54:50.377 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9900 IOs, 3931653120 bytes 2026-02-22T21:54:50.406 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10000 IOs, 3932242944 bytes 2026-02-22T21:54:50.442 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10100 IOs, 3942148096 bytes 2026-02-22T21:54:50.525 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10200 IOs, 4036332544 bytes 2026-02-22T21:54:50.597 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10300 IOs, 4138397696 bytes 2026-02-22T21:54:50.652 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10400 IOs, 4174722048 bytes 2026-02-22T21:54:50.686 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10500 IOs, 4178818048 bytes 2026-02-22T21:54:50.750 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10600 IOs, 4258576384 bytes 2026-02-22T21:54:50.831 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10700 IOs, 4324758528 bytes 2026-02-22T21:54:50.873 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10800 IOs, 4348965888 bytes 2026-02-22T21:54:50.916 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10900 IOs, 4385468416 bytes 2026-02-22T21:54:50.973 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11000 IOs, 4424594432 bytes 2026-02-22T21:54:51.023 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11100 IOs, 4465951744 bytes 2026-02-22T21:54:51.094 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11200 IOs, 4532605952 bytes 2026-02-22T21:54:51.145 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11300 IOs, 4580706304 bytes 2026-02-22T21:54:51.173 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11400 IOs, 4585909248 bytes 2026-02-22T21:54:51.226 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11500 IOs, 4644357120 bytes 2026-02-22T21:54:51.259 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11600 IOs, 4665772032 bytes 2026-02-22T21:54:51.326 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11700 IOs, 4721913856 bytes 2026-02-22T21:54:51.371 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11800 IOs, 4762275840 bytes 2026-02-22T21:54:51.432 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11900 IOs, 4833966080 bytes 2026-02-22T21:54:51.504 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12000 IOs, 4911902720 bytes 2026-02-22T21:54:51.546 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12100 IOs, 4949629952 bytes 2026-02-22T21:54:51.566 INFO:tasks.ceph.cluster2.osd.1.trial026.stderr:2026-02-22T21:54:51.563+0000 7f82fc108640 -1 reset not still connected to 0x55bea6276750 2026-02-22T21:54:51.596 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12200 IOs, 4974334976 bytes 2026-02-22T21:54:51.630 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12300 IOs, 4993713152 bytes 2026-02-22T21:54:51.683 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12400 IOs, 5051961344 bytes 2026-02-22T21:54:51.749 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12500 IOs, 5112290304 bytes 2026-02-22T21:54:51.793 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12600 IOs, 5128350720 bytes 2026-02-22T21:54:51.827 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12700 IOs, 5141183488 bytes 2026-02-22T21:54:51.869 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12800 IOs, 5160569856 bytes 2026-02-22T21:54:51.920 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12900 IOs, 5200856064 bytes 2026-02-22T21:54:51.992 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13000 IOs, 5285723136 bytes 2026-02-22T21:54:52.062 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13100 IOs, 5352956928 bytes 2026-02-22T21:54:52.091 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13200 IOs, 5353581568 bytes 2026-02-22T21:54:52.133 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13300 IOs, 5398731776 bytes 2026-02-22T21:54:52.185 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13400 IOs, 5424230400 bytes 2026-02-22T21:54:52.243 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13500 IOs, 5483330560 bytes 2026-02-22T21:54:52.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13600 IOs, 5534646272 bytes 2026-02-22T21:54:52.328 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13700 IOs, 5544819712 bytes 2026-02-22T21:54:52.379 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13800 IOs, 5584703488 bytes 2026-02-22T21:54:52.433 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13900 IOs, 5610807296 bytes 2026-02-22T21:54:52.474 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14000 IOs, 5657273344 bytes 2026-02-22T21:54:52.535 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14100 IOs, 5710118912 bytes 2026-02-22T21:54:52.616 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14200 IOs, 5809669120 bytes 2026-02-22T21:54:52.679 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14300 IOs, 5877602304 bytes 2026-02-22T21:54:52.736 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14400 IOs, 5925008384 bytes 2026-02-22T21:54:52.797 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14500 IOs, 5978155008 bytes 2026-02-22T21:54:52.842 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14600 IOs, 6008420352 bytes 2026-02-22T21:54:52.903 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14700 IOs, 6060331008 bytes 2026-02-22T21:54:52.941 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14800 IOs, 6089221120 bytes 2026-02-22T21:54:52.983 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14900 IOs, 6099107840 bytes 2026-02-22T21:54:53.030 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15000 IOs, 6133260288 bytes 2026-02-22T21:54:53.088 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15100 IOs, 6191279104 bytes 2026-02-22T21:54:53.344 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15200 IOs, 6256823296 bytes 2026-02-22T21:54:53.413 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15300 IOs, 6316756992 bytes 2026-02-22T21:54:53.469 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15400 IOs, 6389267456 bytes 2026-02-22T21:54:53.532 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15500 IOs, 6414166016 bytes 2026-02-22T21:54:53.566 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15600 IOs, 6425039872 bytes 2026-02-22T21:54:53.661 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15700 IOs, 6553037824 bytes 2026-02-22T21:54:53.759 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15800 IOs, 6686351360 bytes 2026-02-22T21:54:53.804 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15900 IOs, 6717814784 bytes 2026-02-22T21:54:53.884 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16000 IOs, 6799063040 bytes 2026-02-22T21:54:53.944 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16100 IOs, 6855124992 bytes 2026-02-22T21:54:53.990 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16200 IOs, 6876581888 bytes 2026-02-22T21:54:54.040 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16300 IOs, 6915418112 bytes 2026-02-22T21:54:54.078 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16400 IOs, 6931944448 bytes 2026-02-22T21:54:54.147 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16500 IOs, 7022444544 bytes 2026-02-22T21:54:54.183 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16600 IOs, 7053375488 bytes 2026-02-22T21:54:54.203 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ error_code=124 2026-02-22T21:54:54.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set -e 2026-02-22T21:54:54.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' 124 -eq 124 ']' 2026-02-22T21:54:54.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:54:54.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T21:54:54.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_43 2026-02-22T21:54:54.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image_and_enable_mirror cluster2 mirror image_43 snapshot 128M 2026-02-22T21:54:54.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:54:54.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:54:54.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:54:54.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:54:54.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_43 2026-02-22T21:54:54.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:54:54.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T21:54:54.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n snapshot ']' 2026-02-22T21:54:54.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:54:54.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image cluster2 mirror image_43 128M 2026-02-22T21:54:54.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:54:54.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:54:54.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:54:54.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:54:54.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_43 2026-02-22T21:54:54.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:54:54.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local size=128 2026-02-22T21:54:54.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n 128M ']' 2026-02-22T21:54:54.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ size=128M 2026-02-22T21:54:54.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:54:54.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_43 2026-02-22T21:54:54.235 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T21:54:54.235 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:54:54.235 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' image = image ']' 2026-02-22T21:54:54.236 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:54:54.236 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ enable_mirror cluster2 mirror image_43 snapshot 2026-02-22T21:54:54.236 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:54:54.236 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:54:54.236 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_43 2026-02-22T21:54:54.236 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T21:54:54.236 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_43 snapshot 2026-02-22T21:54:55.170 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Mirroring enabled 2026-02-22T21:54:55.175 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 info mirror/image_43 2026-02-22T21:54:55.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_43': 2026-02-22T21:54:55.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 128 MiB in 32 objects 2026-02-22T21:54:55.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T21:54:55.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 1 2026-02-22T21:54:55.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 145dbd8a2e9a 2026-02-22T21:54:55.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.145dbd8a2e9a 2026-02-22T21:54:55.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T21:54:55.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock 2026-02-22T21:54:55.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T21:54:55.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T21:54:55.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:54:54 2026 2026-02-22T21:54:55.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:54:54 2026 2026-02-22T21:54:55.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:54:54 2026 2026-02-22T21:54:55.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T21:54:55.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T21:54:55.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: af18f3fc-e37e-4800-a4fb-c06589d02d67 2026-02-22T21:54:55.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: true 2026-02-22T21:54:55.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n '' ']' 2026-02-22T21:54:55.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ stress_write_image cluster2 mirror image_43 2026-02-22T21:54:55.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:54:55.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:54:55.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_43 2026-02-22T21:54:55.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-22T21:54:55.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local duration=10 2026-02-22T21:54:55.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set +e 2026-02-22T21:54:55.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ timeout 10s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_43 --debug-rbd=20 --debug-journaler=20 2026-02-22T21:54:55.281 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:100 IOs, 42193920 bytes 2026-02-22T21:54:55.321 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:200 IOs, 52981760 bytes 2026-02-22T21:54:55.357 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:300 IOs, 68045824 bytes 2026-02-22T21:54:55.431 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:400 IOs, 153625600 bytes 2026-02-22T21:54:55.482 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:500 IOs, 196919296 bytes 2026-02-22T21:54:55.522 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:600 IOs, 223440896 bytes 2026-02-22T21:54:55.553 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:700 IOs, 224069632 bytes 2026-02-22T21:54:55.600 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:800 IOs, 252434432 bytes 2026-02-22T21:54:55.669 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:900 IOs, 338432000 bytes 2026-02-22T21:54:55.737 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1000 IOs, 380303360 bytes 2026-02-22T21:54:55.766 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1100 IOs, 380917760 bytes 2026-02-22T21:54:55.794 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1200 IOs, 384231424 bytes 2026-02-22T21:54:55.861 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1300 IOs, 452144128 bytes 2026-02-22T21:54:55.921 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1400 IOs, 519417856 bytes 2026-02-22T21:54:55.960 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1500 IOs, 546619392 bytes 2026-02-22T21:54:56.015 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1600 IOs, 595565568 bytes 2026-02-22T21:54:56.048 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1700 IOs, 617077760 bytes 2026-02-22T21:54:56.079 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1800 IOs, 617714688 bytes 2026-02-22T21:54:56.121 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1900 IOs, 649273344 bytes 2026-02-22T21:54:56.170 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2000 IOs, 686123008 bytes 2026-02-22T21:54:56.229 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2100 IOs, 736168960 bytes 2026-02-22T21:54:56.276 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2200 IOs, 767635456 bytes 2026-02-22T21:54:56.338 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2300 IOs, 842417152 bytes 2026-02-22T21:54:56.382 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2400 IOs, 874646528 bytes 2026-02-22T21:54:56.432 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2500 IOs, 896449536 bytes 2026-02-22T21:54:56.473 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2600 IOs, 924066816 bytes 2026-02-22T21:54:56.510 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2700 IOs, 926174208 bytes 2026-02-22T21:54:56.581 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2800 IOs, 1009114112 bytes 2026-02-22T21:54:56.632 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2900 IOs, 1048258560 bytes 2026-02-22T21:54:56.671 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3000 IOs, 1092507648 bytes 2026-02-22T21:54:56.755 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3100 IOs, 1183393792 bytes 2026-02-22T21:54:56.802 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3200 IOs, 1207523328 bytes 2026-02-22T21:54:56.852 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3300 IOs, 1253173248 bytes 2026-02-22T21:54:56.899 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3400 IOs, 1278859264 bytes 2026-02-22T21:54:56.972 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3500 IOs, 1360421888 bytes 2026-02-22T21:54:57.012 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3600 IOs, 1398038528 bytes 2026-02-22T21:54:57.053 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3700 IOs, 1410721792 bytes 2026-02-22T21:54:57.082 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3800 IOs, 1414993920 bytes 2026-02-22T21:54:57.148 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3900 IOs, 1482249216 bytes 2026-02-22T21:54:57.187 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4000 IOs, 1506291712 bytes 2026-02-22T21:54:57.229 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4100 IOs, 1533070336 bytes 2026-02-22T21:54:57.292 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4200 IOs, 1606050816 bytes 2026-02-22T21:54:57.371 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4300 IOs, 1701016576 bytes 2026-02-22T21:54:57.437 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4400 IOs, 1760415744 bytes 2026-02-22T21:54:57.486 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4500 IOs, 1795271680 bytes 2026-02-22T21:54:57.524 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4600 IOs, 1799825408 bytes 2026-02-22T21:54:57.567 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4700 IOs, 1824537600 bytes 2026-02-22T21:54:57.629 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4800 IOs, 1892560896 bytes 2026-02-22T21:54:57.696 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4900 IOs, 1955977216 bytes 2026-02-22T21:54:57.745 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5000 IOs, 1993311232 bytes 2026-02-22T21:54:57.797 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5100 IOs, 2023568384 bytes 2026-02-22T21:54:57.868 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5200 IOs, 2101825536 bytes 2026-02-22T21:54:57.944 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5300 IOs, 2201910272 bytes 2026-02-22T21:54:57.994 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5400 IOs, 2239130624 bytes 2026-02-22T21:54:58.026 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5500 IOs, 2251486208 bytes 2026-02-22T21:54:58.091 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5600 IOs, 2337283072 bytes 2026-02-22T21:54:58.151 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5700 IOs, 2401921024 bytes 2026-02-22T21:54:58.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5800 IOs, 2437274624 bytes 2026-02-22T21:54:58.257 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5900 IOs, 2485147648 bytes 2026-02-22T21:54:58.313 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6000 IOs, 2525510656 bytes 2026-02-22T21:54:58.364 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6100 IOs, 2576972800 bytes 2026-02-22T21:54:58.405 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6200 IOs, 2599207936 bytes 2026-02-22T21:54:58.450 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6300 IOs, 2635835392 bytes 2026-02-22T21:54:58.501 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6400 IOs, 2678882304 bytes 2026-02-22T21:54:58.556 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6500 IOs, 2738851840 bytes 2026-02-22T21:54:58.626 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6600 IOs, 2812535808 bytes 2026-02-22T21:54:58.670 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6700 IOs, 2847559680 bytes 2026-02-22T21:54:58.715 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6800 IOs, 2891788288 bytes 2026-02-22T21:54:58.765 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6900 IOs, 2927217664 bytes 2026-02-22T21:54:58.810 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7000 IOs, 2948001792 bytes 2026-02-22T21:54:58.851 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7100 IOs, 2966910976 bytes 2026-02-22T21:54:58.878 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7200 IOs, 2967542784 bytes 2026-02-22T21:54:58.917 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7300 IOs, 2990503936 bytes 2026-02-22T21:54:58.960 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7400 IOs, 3012480000 bytes 2026-02-22T21:54:59.004 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7500 IOs, 3043445760 bytes 2026-02-22T21:54:59.067 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7600 IOs, 3116672000 bytes 2026-02-22T21:54:59.125 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7700 IOs, 3166285824 bytes 2026-02-22T21:54:59.188 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7800 IOs, 3226748928 bytes 2026-02-22T21:54:59.240 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7900 IOs, 3269847040 bytes 2026-02-22T21:54:59.292 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8000 IOs, 3320226816 bytes 2026-02-22T21:54:59.341 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8100 IOs, 3362522112 bytes 2026-02-22T21:54:59.378 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8200 IOs, 3373700096 bytes 2026-02-22T21:54:59.433 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8300 IOs, 3425825792 bytes 2026-02-22T21:54:59.494 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8400 IOs, 3480297472 bytes 2026-02-22T21:54:59.524 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8500 IOs, 3486094336 bytes 2026-02-22T21:54:59.592 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8600 IOs, 3553773568 bytes 2026-02-22T21:54:59.633 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8700 IOs, 3596756992 bytes 2026-02-22T21:54:59.681 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8800 IOs, 3620075520 bytes 2026-02-22T21:54:59.725 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8900 IOs, 3649929216 bytes 2026-02-22T21:54:59.751 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9000 IOs, 3650554880 bytes 2026-02-22T21:54:59.793 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9100 IOs, 3681628160 bytes 2026-02-22T21:54:59.866 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9200 IOs, 3749192704 bytes 2026-02-22T21:54:59.920 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9300 IOs, 3800622080 bytes 2026-02-22T21:54:59.965 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9400 IOs, 3833338880 bytes 2026-02-22T21:55:00.010 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9500 IOs, 3866329088 bytes 2026-02-22T21:55:00.059 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9600 IOs, 3912156160 bytes 2026-02-22T21:55:00.128 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9700 IOs, 4006714368 bytes 2026-02-22T21:55:00.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9800 IOs, 4040704000 bytes 2026-02-22T21:55:00.233 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9900 IOs, 4077211648 bytes 2026-02-22T21:55:00.293 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10000 IOs, 4137550848 bytes 2026-02-22T21:55:00.335 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10100 IOs, 4160977920 bytes 2026-02-22T21:55:00.413 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10200 IOs, 4272045056 bytes 2026-02-22T21:55:00.463 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10300 IOs, 4298164224 bytes 2026-02-22T21:55:00.520 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10400 IOs, 4330115072 bytes 2026-02-22T21:55:00.574 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10500 IOs, 4374443008 bytes 2026-02-22T21:55:00.620 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10600 IOs, 4411053056 bytes 2026-02-22T21:55:00.661 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10700 IOs, 4438292480 bytes 2026-02-22T21:55:00.701 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10800 IOs, 4475549696 bytes 2026-02-22T21:55:00.754 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10900 IOs, 4522981376 bytes 2026-02-22T21:55:00.800 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11000 IOs, 4554384384 bytes 2026-02-22T21:55:00.869 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11100 IOs, 4617016320 bytes 2026-02-22T21:55:00.927 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11200 IOs, 4692349952 bytes 2026-02-22T21:55:00.976 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11300 IOs, 4720905216 bytes 2026-02-22T21:55:01.010 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11400 IOs, 4732930048 bytes 2026-02-22T21:55:01.061 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11500 IOs, 4772888576 bytes 2026-02-22T21:55:01.110 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11600 IOs, 4813948928 bytes 2026-02-22T21:55:01.155 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11700 IOs, 4837862400 bytes 2026-02-22T21:55:01.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11800 IOs, 4880568320 bytes 2026-02-22T21:55:01.275 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11900 IOs, 4957021184 bytes 2026-02-22T21:55:01.325 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12000 IOs, 5004822528 bytes 2026-02-22T21:55:01.392 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12100 IOs, 5071116288 bytes 2026-02-22T21:55:01.435 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12200 IOs, 5114260480 bytes 2026-02-22T21:55:01.511 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12300 IOs, 5162789888 bytes 2026-02-22T21:55:01.556 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12400 IOs, 5207075840 bytes 2026-02-22T21:55:01.624 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12500 IOs, 5250104320 bytes 2026-02-22T21:55:01.698 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12600 IOs, 5335430144 bytes 2026-02-22T21:55:01.743 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12700 IOs, 5367187456 bytes 2026-02-22T21:55:01.782 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12800 IOs, 5379699712 bytes 2026-02-22T21:55:01.861 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12900 IOs, 5450004480 bytes 2026-02-22T21:55:01.907 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13000 IOs, 5479865344 bytes 2026-02-22T21:55:01.947 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13100 IOs, 5501292544 bytes 2026-02-22T21:55:01.990 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13200 IOs, 5528550400 bytes 2026-02-22T21:55:02.018 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13300 IOs, 5532519424 bytes 2026-02-22T21:55:02.076 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13400 IOs, 5583919104 bytes 2026-02-22T21:55:02.119 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13500 IOs, 5612232704 bytes 2026-02-22T21:55:02.163 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13600 IOs, 5646717952 bytes 2026-02-22T21:55:02.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13700 IOs, 5696337920 bytes 2026-02-22T21:55:02.274 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13800 IOs, 5738023936 bytes 2026-02-22T21:55:02.338 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13900 IOs, 5787997184 bytes 2026-02-22T21:55:02.384 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14000 IOs, 5831128064 bytes 2026-02-22T21:55:02.435 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14100 IOs, 5861023744 bytes 2026-02-22T21:55:02.494 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14200 IOs, 5917684736 bytes 2026-02-22T21:55:02.542 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14300 IOs, 5962218496 bytes 2026-02-22T21:55:02.597 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14400 IOs, 5981752320 bytes 2026-02-22T21:55:02.665 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14500 IOs, 6050574336 bytes 2026-02-22T21:55:02.733 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14600 IOs, 6132902912 bytes 2026-02-22T21:55:02.772 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14700 IOs, 6147397632 bytes 2026-02-22T21:55:02.821 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14800 IOs, 6186830848 bytes 2026-02-22T21:55:02.865 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14900 IOs, 6208966656 bytes 2026-02-22T21:55:02.918 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15000 IOs, 6256556032 bytes 2026-02-22T21:55:03.157 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15100 IOs, 6291140608 bytes 2026-02-22T21:55:03.209 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15200 IOs, 6344181760 bytes 2026-02-22T21:55:03.255 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15300 IOs, 6368451584 bytes 2026-02-22T21:55:03.298 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15400 IOs, 6386321408 bytes 2026-02-22T21:55:03.347 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15500 IOs, 6422004736 bytes 2026-02-22T21:55:03.394 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15600 IOs, 6455727104 bytes 2026-02-22T21:55:03.445 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15700 IOs, 6492487680 bytes 2026-02-22T21:55:03.494 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15800 IOs, 6531153920 bytes 2026-02-22T21:55:03.569 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15900 IOs, 6608948224 bytes 2026-02-22T21:55:03.615 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16000 IOs, 6645438464 bytes 2026-02-22T21:55:03.670 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16100 IOs, 6711375872 bytes 2026-02-22T21:55:03.758 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16200 IOs, 6813562880 bytes 2026-02-22T21:55:03.811 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16300 IOs, 6856916992 bytes 2026-02-22T21:55:03.857 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16400 IOs, 6886332416 bytes 2026-02-22T21:55:03.928 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16500 IOs, 6946497536 bytes 2026-02-22T21:55:03.985 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16600 IOs, 7001394176 bytes 2026-02-22T21:55:04.048 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16700 IOs, 7083701248 bytes 2026-02-22T21:55:04.090 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16800 IOs, 7102537728 bytes 2026-02-22T21:55:04.146 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16900 IOs, 7166547968 bytes 2026-02-22T21:55:04.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17000 IOs, 7201818624 bytes 2026-02-22T21:55:04.236 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17100 IOs, 7213669376 bytes 2026-02-22T21:55:04.269 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17200 IOs, 7232380928 bytes 2026-02-22T21:55:04.325 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17300 IOs, 7275316224 bytes 2026-02-22T21:55:04.387 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17400 IOs, 7350859776 bytes 2026-02-22T21:55:04.426 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17500 IOs, 7364647936 bytes 2026-02-22T21:55:04.466 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17600 IOs, 7391385600 bytes 2026-02-22T21:55:04.525 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17700 IOs, 7472173056 bytes 2026-02-22T21:55:04.593 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17800 IOs, 7510704128 bytes 2026-02-22T21:55:04.620 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17900 IOs, 7511325696 bytes 2026-02-22T21:55:04.662 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18000 IOs, 7541935104 bytes 2026-02-22T21:55:04.709 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18100 IOs, 7591089152 bytes 2026-02-22T21:55:04.764 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18200 IOs, 7633017856 bytes 2026-02-22T21:55:04.832 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18300 IOs, 7692407808 bytes 2026-02-22T21:55:04.887 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18400 IOs, 7722498048 bytes 2026-02-22T21:55:04.942 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18500 IOs, 7764475904 bytes 2026-02-22T21:55:04.975 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18600 IOs, 7773474816 bytes 2026-02-22T21:55:05.022 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18700 IOs, 7812483072 bytes 2026-02-22T21:55:05.072 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18800 IOs, 7848739840 bytes 2026-02-22T21:55:05.117 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18900 IOs, 7883316224 bytes 2026-02-22T21:55:05.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19000 IOs, 7955656704 bytes 2026-02-22T21:55:05.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ error_code=124 2026-02-22T21:55:05.209 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set -e 2026-02-22T21:55:05.209 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' 124 -eq 124 ']' 2026-02-22T21:55:05.209 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:55:05.209 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T21:55:05.209 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_44 2026-02-22T21:55:05.209 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image_and_enable_mirror cluster2 mirror image_44 snapshot 128M 2026-02-22T21:55:05.209 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:55:05.209 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:55:05.209 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:55:05.209 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:55:05.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_44 2026-02-22T21:55:05.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:55:05.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T21:55:05.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n snapshot ']' 2026-02-22T21:55:05.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:55:05.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image cluster2 mirror image_44 128M 2026-02-22T21:55:05.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:55:05.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:55:05.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:55:05.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:55:05.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_44 2026-02-22T21:55:05.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:55:05.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local size=128 2026-02-22T21:55:05.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n 128M ']' 2026-02-22T21:55:05.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ size=128M 2026-02-22T21:55:05.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:55:05.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_44 2026-02-22T21:55:05.240 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T21:55:05.241 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:55:05.241 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' image = image ']' 2026-02-22T21:55:05.241 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:55:05.241 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ enable_mirror cluster2 mirror image_44 snapshot 2026-02-22T21:55:05.241 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:55:05.241 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:55:05.242 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_44 2026-02-22T21:55:05.242 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T21:55:05.242 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_44 snapshot 2026-02-22T21:55:06.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Mirroring enabled 2026-02-22T21:55:06.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 info mirror/image_44 2026-02-22T21:55:06.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_44': 2026-02-22T21:55:06.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 128 MiB in 32 objects 2026-02-22T21:55:06.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T21:55:06.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 1 2026-02-22T21:55:06.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 146658877350 2026-02-22T21:55:06.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.146658877350 2026-02-22T21:55:06.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T21:55:06.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock 2026-02-22T21:55:06.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T21:55:06.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T21:55:06.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:55:05 2026 2026-02-22T21:55:06.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:55:05 2026 2026-02-22T21:55:06.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:55:05 2026 2026-02-22T21:55:06.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T21:55:06.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T21:55:06.209 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: 1f09f5dc-1ede-4e63-bd97-5b2825b46268 2026-02-22T21:55:06.209 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: true 2026-02-22T21:55:06.209 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n '' ']' 2026-02-22T21:55:06.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ stress_write_image cluster2 mirror image_44 2026-02-22T21:55:06.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:55:06.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:55:06.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_44 2026-02-22T21:55:06.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-22T21:55:06.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local duration=10 2026-02-22T21:55:06.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set +e 2026-02-22T21:55:06.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ timeout 10s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_44 --debug-rbd=20 --debug-journaler=20 2026-02-22T21:55:06.310 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:100 IOs, 64926720 bytes 2026-02-22T21:55:06.363 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:200 IOs, 115363840 bytes 2026-02-22T21:55:06.414 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:300 IOs, 151484416 bytes 2026-02-22T21:55:06.480 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:400 IOs, 224760832 bytes 2026-02-22T21:55:06.514 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:500 IOs, 241125376 bytes 2026-02-22T21:55:06.557 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:600 IOs, 264977408 bytes 2026-02-22T21:55:06.618 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:700 IOs, 336914432 bytes 2026-02-22T21:55:06.682 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:800 IOs, 414283776 bytes 2026-02-22T21:55:06.730 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:900 IOs, 441679872 bytes 2026-02-22T21:55:06.771 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1000 IOs, 472243200 bytes 2026-02-22T21:55:06.806 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1100 IOs, 482128896 bytes 2026-02-22T21:55:06.850 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1200 IOs, 512717824 bytes 2026-02-22T21:55:06.918 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1300 IOs, 570232832 bytes 2026-02-22T21:55:06.955 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1400 IOs, 584360960 bytes 2026-02-22T21:55:07.016 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1500 IOs, 640449536 bytes 2026-02-22T21:55:07.084 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1600 IOs, 688081920 bytes 2026-02-22T21:55:07.140 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1700 IOs, 740092928 bytes 2026-02-22T21:55:07.203 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1800 IOs, 793731072 bytes 2026-02-22T21:55:07.247 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1900 IOs, 822155264 bytes 2026-02-22T21:55:07.339 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2000 IOs, 955518976 bytes 2026-02-22T21:55:07.409 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2100 IOs, 1017582592 bytes 2026-02-22T21:55:07.437 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2200 IOs, 1018716160 bytes 2026-02-22T21:55:07.495 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2300 IOs, 1075572736 bytes 2026-02-22T21:55:07.553 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2400 IOs, 1126311936 bytes 2026-02-22T21:55:07.601 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2500 IOs, 1169878016 bytes 2026-02-22T21:55:07.650 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2600 IOs, 1210225664 bytes 2026-02-22T21:55:07.701 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2700 IOs, 1256843264 bytes 2026-02-22T21:55:07.757 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2800 IOs, 1291377664 bytes 2026-02-22T21:55:07.817 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2900 IOs, 1380451328 bytes 2026-02-22T21:55:07.870 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3000 IOs, 1418669056 bytes 2026-02-22T21:55:07.919 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3100 IOs, 1443086336 bytes 2026-02-22T21:55:07.985 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3200 IOs, 1513419776 bytes 2026-02-22T21:55:08.039 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3300 IOs, 1574358016 bytes 2026-02-22T21:55:08.090 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3400 IOs, 1614447616 bytes 2026-02-22T21:55:08.142 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3500 IOs, 1663802368 bytes 2026-02-22T21:55:08.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3600 IOs, 1665329152 bytes 2026-02-22T21:55:08.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3700 IOs, 1665950720 bytes 2026-02-22T21:55:08.246 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3800 IOs, 1694291968 bytes 2026-02-22T21:55:08.289 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3900 IOs, 1730574336 bytes 2026-02-22T21:55:08.333 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4000 IOs, 1765158912 bytes 2026-02-22T21:55:08.415 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4100 IOs, 1848898560 bytes 2026-02-22T21:55:08.463 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4200 IOs, 1893681152 bytes 2026-02-22T21:55:08.517 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4300 IOs, 1927983104 bytes 2026-02-22T21:55:08.558 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4400 IOs, 1948994560 bytes 2026-02-22T21:55:08.616 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4500 IOs, 2006139904 bytes 2026-02-22T21:55:08.667 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4600 IOs, 2050578432 bytes 2026-02-22T21:55:08.717 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4700 IOs, 2085199872 bytes 2026-02-22T21:55:08.769 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4800 IOs, 2146069504 bytes 2026-02-22T21:55:08.813 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4900 IOs, 2163982336 bytes 2026-02-22T21:55:08.853 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5000 IOs, 2197149696 bytes 2026-02-22T21:55:08.901 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5100 IOs, 2217737216 bytes 2026-02-22T21:55:08.953 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5200 IOs, 2258065408 bytes 2026-02-22T21:55:09.000 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5300 IOs, 2274677760 bytes 2026-02-22T21:55:09.057 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5400 IOs, 2341178368 bytes 2026-02-22T21:55:09.098 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5500 IOs, 2341803008 bytes 2026-02-22T21:55:09.159 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5600 IOs, 2412009472 bytes 2026-02-22T21:55:09.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5700 IOs, 2462300160 bytes 2026-02-22T21:55:09.244 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5800 IOs, 2466463744 bytes 2026-02-22T21:55:09.294 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5900 IOs, 2513652736 bytes 2026-02-22T21:55:09.339 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6000 IOs, 2555516928 bytes 2026-02-22T21:55:09.401 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6100 IOs, 2610256896 bytes 2026-02-22T21:55:09.448 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6200 IOs, 2651318272 bytes 2026-02-22T21:55:09.508 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6300 IOs, 2697527296 bytes 2026-02-22T21:55:09.543 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6400 IOs, 2716787712 bytes 2026-02-22T21:55:09.577 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6500 IOs, 2731940864 bytes 2026-02-22T21:55:09.630 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6600 IOs, 2781768704 bytes 2026-02-22T21:55:09.668 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6700 IOs, 2809828352 bytes 2026-02-22T21:55:09.719 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6800 IOs, 2851570688 bytes 2026-02-22T21:55:09.769 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6900 IOs, 2894562304 bytes 2026-02-22T21:55:09.816 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7000 IOs, 2918650880 bytes 2026-02-22T21:55:09.881 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7100 IOs, 2995247104 bytes 2026-02-22T21:55:09.940 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7200 IOs, 3048925184 bytes 2026-02-22T21:55:10.011 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7300 IOs, 3146240000 bytes 2026-02-22T21:55:10.067 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7400 IOs, 3203264512 bytes 2026-02-22T21:55:10.147 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7500 IOs, 3277651968 bytes 2026-02-22T21:55:10.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7600 IOs, 3331358720 bytes 2026-02-22T21:55:10.251 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7700 IOs, 3360262144 bytes 2026-02-22T21:55:10.293 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7800 IOs, 3386917888 bytes 2026-02-22T21:55:10.358 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7900 IOs, 3455630336 bytes 2026-02-22T21:55:10.406 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8000 IOs, 3500208128 bytes 2026-02-22T21:55:10.447 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8100 IOs, 3521942528 bytes 2026-02-22T21:55:10.529 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8200 IOs, 3648386048 bytes 2026-02-22T21:55:10.577 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8300 IOs, 3680347136 bytes 2026-02-22T21:55:10.635 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8400 IOs, 3735367680 bytes 2026-02-22T21:55:10.706 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8500 IOs, 3826640896 bytes 2026-02-22T21:55:10.771 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8600 IOs, 3895593984 bytes 2026-02-22T21:55:10.801 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8700 IOs, 3898160128 bytes 2026-02-22T21:55:10.840 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8800 IOs, 3922163712 bytes 2026-02-22T21:55:10.898 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8900 IOs, 3978560512 bytes 2026-02-22T21:55:10.984 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9000 IOs, 4071561216 bytes 2026-02-22T21:55:11.022 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9100 IOs, 4094561280 bytes 2026-02-22T21:55:11.069 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9200 IOs, 4124530688 bytes 2026-02-22T21:55:11.127 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9300 IOs, 4181938176 bytes 2026-02-22T21:55:11.162 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9400 IOs, 4195156992 bytes 2026-02-22T21:55:11.209 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9500 IOs, 4226965504 bytes 2026-02-22T21:55:11.244 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9600 IOs, 4239892480 bytes 2026-02-22T21:55:11.306 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9700 IOs, 4321882112 bytes 2026-02-22T21:55:11.367 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9800 IOs, 4384532480 bytes 2026-02-22T21:55:11.433 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9900 IOs, 4451968000 bytes 2026-02-22T21:55:11.465 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10000 IOs, 4461700096 bytes 2026-02-22T21:55:11.535 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10100 IOs, 4548182016 bytes 2026-02-22T21:55:11.585 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10200 IOs, 4575252480 bytes 2026-02-22T21:55:11.646 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10300 IOs, 4653306880 bytes 2026-02-22T21:55:11.723 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10400 IOs, 4723787776 bytes 2026-02-22T21:55:11.757 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10500 IOs, 4742458368 bytes 2026-02-22T21:55:11.793 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10600 IOs, 4750193664 bytes 2026-02-22T21:55:11.862 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10700 IOs, 4827793408 bytes 2026-02-22T21:55:11.909 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10800 IOs, 4868624384 bytes 2026-02-22T21:55:11.953 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10900 IOs, 4884850688 bytes 2026-02-22T21:55:12.027 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11000 IOs, 4964450304 bytes 2026-02-22T21:55:12.057 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11100 IOs, 4971273216 bytes 2026-02-22T21:55:12.092 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11200 IOs, 4987780096 bytes 2026-02-22T21:55:12.139 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11300 IOs, 5030666240 bytes 2026-02-22T21:55:12.222 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11400 IOs, 5141315584 bytes 2026-02-22T21:55:12.274 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11500 IOs, 5181117440 bytes 2026-02-22T21:55:12.317 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11600 IOs, 5205730304 bytes 2026-02-22T21:55:12.360 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11700 IOs, 5231418368 bytes 2026-02-22T21:55:12.408 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11800 IOs, 5268052992 bytes 2026-02-22T21:55:12.443 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11900 IOs, 5291205632 bytes 2026-02-22T21:55:12.507 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12000 IOs, 5361954816 bytes 2026-02-22T21:55:12.578 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12100 IOs, 5432130560 bytes 2026-02-22T21:55:12.633 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12200 IOs, 5474312192 bytes 2026-02-22T21:55:12.684 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12300 IOs, 5536501760 bytes 2026-02-22T21:55:12.744 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12400 IOs, 5587937280 bytes 2026-02-22T21:55:12.787 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12500 IOs, 5619537920 bytes 2026-02-22T21:55:12.845 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12600 IOs, 5688440832 bytes 2026-02-22T21:55:12.910 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12700 IOs, 5745571840 bytes 2026-02-22T21:55:12.985 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12800 IOs, 5820521472 bytes 2026-02-22T21:55:13.026 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12900 IOs, 5842081792 bytes 2026-02-22T21:55:13.078 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13000 IOs, 5904115712 bytes 2026-02-22T21:55:13.173 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13100 IOs, 6007254016 bytes 2026-02-22T21:55:13.226 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13200 IOs, 6050779136 bytes 2026-02-22T21:55:13.291 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13300 IOs, 6122722304 bytes 2026-02-22T21:55:13.332 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13400 IOs, 6153929728 bytes 2026-02-22T21:55:13.370 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13500 IOs, 6177726464 bytes 2026-02-22T21:55:13.423 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13600 IOs, 6245623808 bytes 2026-02-22T21:55:13.483 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13700 IOs, 6275402752 bytes 2026-02-22T21:55:13.520 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13800 IOs, 6291036160 bytes 2026-02-22T21:55:13.565 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13900 IOs, 6321054720 bytes 2026-02-22T21:55:13.627 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14000 IOs, 6381694976 bytes 2026-02-22T21:55:13.685 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14100 IOs, 6434744320 bytes 2026-02-22T21:55:13.742 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14200 IOs, 6490557440 bytes 2026-02-22T21:55:13.785 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14300 IOs, 6528884736 bytes 2026-02-22T21:55:13.836 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14400 IOs, 6582889472 bytes 2026-02-22T21:55:13.900 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14500 IOs, 6636476416 bytes 2026-02-22T21:55:13.953 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14600 IOs, 6683372544 bytes 2026-02-22T21:55:14.003 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14700 IOs, 6722607104 bytes 2026-02-22T21:55:14.045 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14800 IOs, 6733157376 bytes 2026-02-22T21:55:14.074 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14900 IOs, 6736273408 bytes 2026-02-22T21:55:14.109 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15000 IOs, 6749574144 bytes 2026-02-22T21:55:14.183 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15100 IOs, 6833282048 bytes 2026-02-22T21:55:14.222 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15200 IOs, 6851076096 bytes 2026-02-22T21:55:14.277 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15300 IOs, 6905519104 bytes 2026-02-22T21:55:14.340 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15400 IOs, 6978899968 bytes 2026-02-22T21:55:14.381 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15500 IOs, 7005022208 bytes 2026-02-22T21:55:14.425 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15600 IOs, 7033246720 bytes 2026-02-22T21:55:14.475 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15700 IOs, 7074964480 bytes 2026-02-22T21:55:14.508 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15800 IOs, 7089903616 bytes 2026-02-22T21:55:14.570 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15900 IOs, 7152930816 bytes 2026-02-22T21:55:14.607 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16000 IOs, 7171905536 bytes 2026-02-22T21:55:14.660 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16100 IOs, 7217806336 bytes 2026-02-22T21:55:14.710 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16200 IOs, 7254598656 bytes 2026-02-22T21:55:14.759 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16300 IOs, 7297641472 bytes 2026-02-22T21:55:14.821 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16400 IOs, 7360473088 bytes 2026-02-22T21:55:14.861 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16500 IOs, 7385654272 bytes 2026-02-22T21:55:14.918 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16600 IOs, 7443507200 bytes 2026-02-22T21:55:14.950 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16700 IOs, 7444115456 bytes 2026-02-22T21:55:14.991 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16800 IOs, 7463543808 bytes 2026-02-22T21:55:15.045 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16900 IOs, 7533660160 bytes 2026-02-22T21:55:15.112 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17000 IOs, 7589155840 bytes 2026-02-22T21:55:15.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17100 IOs, 7632594944 bytes 2026-02-22T21:55:15.195 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17200 IOs, 7642694656 bytes 2026-02-22T21:55:15.276 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17300 IOs, 7740518400 bytes 2026-02-22T21:55:15.329 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17400 IOs, 7775415296 bytes 2026-02-22T21:55:15.362 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17500 IOs, 7789658112 bytes 2026-02-22T21:55:15.401 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17600 IOs, 7808952320 bytes 2026-02-22T21:55:15.448 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17700 IOs, 7834307584 bytes 2026-02-22T21:55:15.505 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17800 IOs, 7894984704 bytes 2026-02-22T21:55:15.554 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17900 IOs, 7936343040 bytes 2026-02-22T21:55:15.595 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18000 IOs, 7956322304 bytes 2026-02-22T21:55:15.639 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18100 IOs, 7982531584 bytes 2026-02-22T21:55:15.680 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18200 IOs, 8004757504 bytes 2026-02-22T21:55:15.725 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18300 IOs, 8046168064 bytes 2026-02-22T21:55:15.800 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18400 IOs, 8111280128 bytes 2026-02-22T21:55:15.851 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18500 IOs, 8161816576 bytes 2026-02-22T21:55:15.920 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18600 IOs, 8248798208 bytes 2026-02-22T21:55:15.971 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18700 IOs, 8267049984 bytes 2026-02-22T21:55:16.013 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18800 IOs, 8290721792 bytes 2026-02-22T21:55:16.053 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18900 IOs, 8318081024 bytes 2026-02-22T21:55:16.112 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19000 IOs, 8374871040 bytes 2026-02-22T21:55:16.139 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19100 IOs, 8375471104 bytes 2026-02-22T21:55:16.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19200 IOs, 8442290176 bytes 2026-02-22T21:55:16.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ error_code=124 2026-02-22T21:55:16.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set -e 2026-02-22T21:55:16.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' 124 -eq 124 ']' 2026-02-22T21:55:16.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:55:16.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T21:55:16.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_45 2026-02-22T21:55:16.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image_and_enable_mirror cluster2 mirror image_45 snapshot 128M 2026-02-22T21:55:16.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:55:16.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:55:16.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:55:16.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:55:16.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_45 2026-02-22T21:55:16.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:55:16.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T21:55:16.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n snapshot ']' 2026-02-22T21:55:16.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:55:16.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image cluster2 mirror image_45 128M 2026-02-22T21:55:16.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:55:16.218 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:55:16.218 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:55:16.218 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:55:16.218 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_45 2026-02-22T21:55:16.218 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:55:16.218 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local size=128 2026-02-22T21:55:16.218 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n 128M ']' 2026-02-22T21:55:16.218 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ size=128M 2026-02-22T21:55:16.218 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:55:16.218 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_45 2026-02-22T21:55:16.246 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T21:55:16.246 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:55:16.246 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' image = image ']' 2026-02-22T21:55:16.246 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:55:16.246 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ enable_mirror cluster2 mirror image_45 snapshot 2026-02-22T21:55:16.246 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:55:16.246 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:55:16.246 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_45 2026-02-22T21:55:16.246 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T21:55:16.246 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_45 snapshot 2026-02-22T21:55:17.143 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Mirroring enabled 2026-02-22T21:55:17.148 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 info mirror/image_45 2026-02-22T21:55:17.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_45': 2026-02-22T21:55:17.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 128 MiB in 32 objects 2026-02-22T21:55:17.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T21:55:17.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 1 2026-02-22T21:55:17.173 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 1470d9b31b5a 2026-02-22T21:55:17.173 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.1470d9b31b5a 2026-02-22T21:55:17.173 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T21:55:17.173 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock 2026-02-22T21:55:17.173 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T21:55:17.173 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T21:55:17.173 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:55:16 2026 2026-02-22T21:55:17.173 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:55:16 2026 2026-02-22T21:55:17.173 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:55:16 2026 2026-02-22T21:55:17.173 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T21:55:17.174 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T21:55:17.174 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: 1024109e-7bb5-4ed3-ab78-ad539d98c9fd 2026-02-22T21:55:17.174 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: true 2026-02-22T21:55:17.174 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n '' ']' 2026-02-22T21:55:17.175 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ stress_write_image cluster2 mirror image_45 2026-02-22T21:55:17.175 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:55:17.175 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:55:17.175 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_45 2026-02-22T21:55:17.175 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-22T21:55:17.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local duration=13 2026-02-22T21:55:17.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set +e 2026-02-22T21:55:17.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ timeout 13s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_45 --debug-rbd=20 --debug-journaler=20 2026-02-22T21:55:17.258 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:100 IOs, 25396224 bytes 2026-02-22T21:55:17.306 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:200 IOs, 68894720 bytes 2026-02-22T21:55:17.364 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:300 IOs, 107495424 bytes 2026-02-22T21:55:17.426 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:400 IOs, 164595712 bytes 2026-02-22T21:55:17.470 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:500 IOs, 190963712 bytes 2026-02-22T21:55:17.498 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:600 IOs, 191571968 bytes 2026-02-22T21:55:17.544 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:700 IOs, 220695552 bytes 2026-02-22T21:55:17.594 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:800 IOs, 286057472 bytes 2026-02-22T21:55:17.638 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:900 IOs, 299539456 bytes 2026-02-22T21:55:17.686 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1000 IOs, 327213056 bytes 2026-02-22T21:55:17.733 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1100 IOs, 359578624 bytes 2026-02-22T21:55:17.767 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1200 IOs, 370290688 bytes 2026-02-22T21:55:17.849 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1300 IOs, 485535744 bytes 2026-02-22T21:55:17.928 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1400 IOs, 552464384 bytes 2026-02-22T21:55:18.007 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1500 IOs, 640132096 bytes 2026-02-22T21:55:18.056 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1600 IOs, 673115136 bytes 2026-02-22T21:55:18.105 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1700 IOs, 720423936 bytes 2026-02-22T21:55:18.154 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1800 IOs, 758664192 bytes 2026-02-22T21:55:18.185 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1900 IOs, 763208704 bytes 2026-02-22T21:55:18.229 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2000 IOs, 792856576 bytes 2026-02-22T21:55:18.256 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2100 IOs, 793487360 bytes 2026-02-22T21:55:18.308 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2200 IOs, 828397568 bytes 2026-02-22T21:55:18.335 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2300 IOs, 828989440 bytes 2026-02-22T21:55:18.395 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2400 IOs, 896081920 bytes 2026-02-22T21:55:18.435 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2500 IOs, 911968256 bytes 2026-02-22T21:55:18.478 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2600 IOs, 936030208 bytes 2026-02-22T21:55:18.531 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2700 IOs, 994383872 bytes 2026-02-22T21:55:18.578 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2800 IOs, 1036289024 bytes 2026-02-22T21:55:18.625 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2900 IOs, 1058965504 bytes 2026-02-22T21:55:18.680 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3000 IOs, 1108877312 bytes 2026-02-22T21:55:18.746 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3100 IOs, 1177538560 bytes 2026-02-22T21:55:18.847 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3200 IOs, 1316959232 bytes 2026-02-22T21:55:18.897 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3300 IOs, 1353042944 bytes 2026-02-22T21:55:18.936 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3400 IOs, 1367152640 bytes 2026-02-22T21:55:18.980 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3500 IOs, 1391940608 bytes 2026-02-22T21:55:19.035 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3600 IOs, 1431751680 bytes 2026-02-22T21:55:19.111 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3700 IOs, 1525486592 bytes 2026-02-22T21:55:19.184 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3800 IOs, 1614430208 bytes 2026-02-22T21:55:19.227 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3900 IOs, 1629524992 bytes 2026-02-22T21:55:19.269 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4000 IOs, 1660007424 bytes 2026-02-22T21:55:19.324 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4100 IOs, 1707233280 bytes 2026-02-22T21:55:19.366 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4200 IOs, 1756414976 bytes 2026-02-22T21:55:19.442 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4300 IOs, 1823921152 bytes 2026-02-22T21:55:19.484 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4400 IOs, 1853048832 bytes 2026-02-22T21:55:19.520 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4500 IOs, 1876264960 bytes 2026-02-22T21:55:19.573 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4600 IOs, 1928838144 bytes 2026-02-22T21:55:19.636 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4700 IOs, 2007328768 bytes 2026-02-22T21:55:19.690 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4800 IOs, 2031714304 bytes 2026-02-22T21:55:19.735 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4900 IOs, 2068556800 bytes 2026-02-22T21:55:19.782 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5000 IOs, 2109333504 bytes 2026-02-22T21:55:19.819 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5100 IOs, 2120590336 bytes 2026-02-22T21:55:19.887 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5200 IOs, 2175811584 bytes 2026-02-22T21:55:19.947 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5300 IOs, 2247835648 bytes 2026-02-22T21:55:20.023 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5400 IOs, 2302886912 bytes 2026-02-22T21:55:20.092 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5500 IOs, 2370936832 bytes 2026-02-22T21:55:20.142 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5600 IOs, 2414856192 bytes 2026-02-22T21:55:20.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5700 IOs, 2441019392 bytes 2026-02-22T21:55:20.231 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5800 IOs, 2469003264 bytes 2026-02-22T21:55:20.257 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5900 IOs, 2469603328 bytes 2026-02-22T21:55:20.295 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6000 IOs, 2498855936 bytes 2026-02-22T21:55:20.341 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6100 IOs, 2536265728 bytes 2026-02-22T21:55:20.393 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6200 IOs, 2568889344 bytes 2026-02-22T21:55:20.419 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6300 IOs, 2569487360 bytes 2026-02-22T21:55:20.472 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6400 IOs, 2610605056 bytes 2026-02-22T21:55:20.705 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6500 IOs, 2623531008 bytes 2026-02-22T21:55:20.763 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6600 IOs, 2671972352 bytes 2026-02-22T21:55:20.821 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6700 IOs, 2725498880 bytes 2026-02-22T21:55:20.862 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6800 IOs, 2745789440 bytes 2026-02-22T21:55:20.922 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6900 IOs, 2810937344 bytes 2026-02-22T21:55:20.972 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7000 IOs, 2829410304 bytes 2026-02-22T21:55:21.031 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7100 IOs, 2883591168 bytes 2026-02-22T21:55:21.093 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7200 IOs, 2956597248 bytes 2026-02-22T21:55:21.160 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7300 IOs, 3006350336 bytes 2026-02-22T21:55:21.187 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7400 IOs, 3014968320 bytes 2026-02-22T21:55:21.214 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7500 IOs, 3015589888 bytes 2026-02-22T21:55:21.249 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7600 IOs, 3037757440 bytes 2026-02-22T21:55:21.285 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7700 IOs, 3057545216 bytes 2026-02-22T21:55:21.361 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7800 IOs, 3139589120 bytes 2026-02-22T21:55:21.449 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7900 IOs, 3241159680 bytes 2026-02-22T21:55:21.511 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8000 IOs, 3308442624 bytes 2026-02-22T21:55:21.572 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8100 IOs, 3368822784 bytes 2026-02-22T21:55:21.644 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8200 IOs, 3452192768 bytes 2026-02-22T21:55:21.695 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8300 IOs, 3499994112 bytes 2026-02-22T21:55:21.736 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8400 IOs, 3519241216 bytes 2026-02-22T21:55:21.795 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8500 IOs, 3591399424 bytes 2026-02-22T21:55:21.842 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8600 IOs, 3612250112 bytes 2026-02-22T21:55:21.904 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8700 IOs, 3677258752 bytes 2026-02-22T21:55:21.956 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8800 IOs, 3721820160 bytes 2026-02-22T21:55:22.021 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8900 IOs, 3770151936 bytes 2026-02-22T21:55:22.082 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9000 IOs, 3828931584 bytes 2026-02-22T21:55:22.139 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9100 IOs, 3883901952 bytes 2026-02-22T21:55:22.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9200 IOs, 3930185728 bytes 2026-02-22T21:55:22.276 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9300 IOs, 4012155904 bytes 2026-02-22T21:55:22.307 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9400 IOs, 4012753920 bytes 2026-02-22T21:55:22.354 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9500 IOs, 4052273152 bytes 2026-02-22T21:55:22.423 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9600 IOs, 4127589376 bytes 2026-02-22T21:55:22.482 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9700 IOs, 4177909760 bytes 2026-02-22T21:55:22.511 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9800 IOs, 4180271104 bytes 2026-02-22T21:55:22.565 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9900 IOs, 4244034560 bytes 2026-02-22T21:55:22.624 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10000 IOs, 4292767744 bytes 2026-02-22T21:55:22.683 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10100 IOs, 4339376128 bytes 2026-02-22T21:55:22.723 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10200 IOs, 4362135552 bytes 2026-02-22T21:55:22.764 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10300 IOs, 4384006144 bytes 2026-02-22T21:55:22.800 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10400 IOs, 4400640000 bytes 2026-02-22T21:55:22.861 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10500 IOs, 4452692992 bytes 2026-02-22T21:55:22.890 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10600 IOs, 4456865792 bytes 2026-02-22T21:55:22.962 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10700 IOs, 4557404160 bytes 2026-02-22T21:55:23.031 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10800 IOs, 4612314112 bytes 2026-02-22T21:55:23.080 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10900 IOs, 4636279808 bytes 2026-02-22T21:55:23.135 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11000 IOs, 4693156864 bytes 2026-02-22T21:55:23.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11100 IOs, 4705003520 bytes 2026-02-22T21:55:23.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11200 IOs, 4718914560 bytes 2026-02-22T21:55:23.255 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11300 IOs, 4762576896 bytes 2026-02-22T21:55:23.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11400 IOs, 4763196416 bytes 2026-02-22T21:55:23.340 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11500 IOs, 4814814208 bytes 2026-02-22T21:55:23.408 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11600 IOs, 4873394176 bytes 2026-02-22T21:55:23.441 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11700 IOs, 4884241408 bytes 2026-02-22T21:55:23.491 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11800 IOs, 4918897664 bytes 2026-02-22T21:55:23.572 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11900 IOs, 5011156992 bytes 2026-02-22T21:55:23.609 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12000 IOs, 5042823168 bytes 2026-02-22T21:55:23.656 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12100 IOs, 5078618112 bytes 2026-02-22T21:55:23.697 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12200 IOs, 5087647744 bytes 2026-02-22T21:55:23.740 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12300 IOs, 5113469952 bytes 2026-02-22T21:55:23.804 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12400 IOs, 5171060736 bytes 2026-02-22T21:55:23.870 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12500 IOs, 5258976256 bytes 2026-02-22T21:55:23.935 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12600 IOs, 5307717632 bytes 2026-02-22T21:55:23.994 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12700 IOs, 5353166848 bytes 2026-02-22T21:55:24.052 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12800 IOs, 5379966976 bytes 2026-02-22T21:55:24.095 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12900 IOs, 5404155904 bytes 2026-02-22T21:55:24.144 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13000 IOs, 5438264320 bytes 2026-02-22T21:55:24.213 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13100 IOs, 5515207680 bytes 2026-02-22T21:55:24.248 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13200 IOs, 5515828224 bytes 2026-02-22T21:55:24.309 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13300 IOs, 5580352512 bytes 2026-02-22T21:55:24.362 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13400 IOs, 5616206848 bytes 2026-02-22T21:55:24.418 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13500 IOs, 5661888512 bytes 2026-02-22T21:55:24.478 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13600 IOs, 5720758272 bytes 2026-02-22T21:55:24.545 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13700 IOs, 5769646080 bytes 2026-02-22T21:55:24.575 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13800 IOs, 5791456256 bytes 2026-02-22T21:55:24.623 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13900 IOs, 5840629760 bytes 2026-02-22T21:55:24.700 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14000 IOs, 5919821824 bytes 2026-02-22T21:55:24.756 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14100 IOs, 5955369984 bytes 2026-02-22T21:55:24.782 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14200 IOs, 5962041344 bytes 2026-02-22T21:55:24.842 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14300 IOs, 6016699392 bytes 2026-02-22T21:55:24.886 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14400 IOs, 6047579136 bytes 2026-02-22T21:55:24.941 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14500 IOs, 6090984448 bytes 2026-02-22T21:55:24.995 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14600 IOs, 6128946176 bytes 2026-02-22T21:55:25.077 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14700 IOs, 6260257792 bytes 2026-02-22T21:55:25.132 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14800 IOs, 6288244736 bytes 2026-02-22T21:55:25.169 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14900 IOs, 6305821696 bytes 2026-02-22T21:55:25.214 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15000 IOs, 6348638208 bytes 2026-02-22T21:55:25.258 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15100 IOs, 6378713088 bytes 2026-02-22T21:55:25.308 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15200 IOs, 6428465152 bytes 2026-02-22T21:55:25.351 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15300 IOs, 6446188544 bytes 2026-02-22T21:55:25.399 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15400 IOs, 6488965120 bytes 2026-02-22T21:55:25.468 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15500 IOs, 6569077760 bytes 2026-02-22T21:55:25.511 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15600 IOs, 6584562688 bytes 2026-02-22T21:55:25.547 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15700 IOs, 6596804608 bytes 2026-02-22T21:55:25.615 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15800 IOs, 6661270528 bytes 2026-02-22T21:55:25.654 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15900 IOs, 6683145216 bytes 2026-02-22T21:55:25.702 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16000 IOs, 6714392576 bytes 2026-02-22T21:55:25.740 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16100 IOs, 6733164544 bytes 2026-02-22T21:55:25.794 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16200 IOs, 6779035648 bytes 2026-02-22T21:55:25.821 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16300 IOs, 6779659264 bytes 2026-02-22T21:55:25.854 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16400 IOs, 6802807808 bytes 2026-02-22T21:55:25.888 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16500 IOs, 6805711872 bytes 2026-02-22T21:55:25.955 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16600 IOs, 6884237312 bytes 2026-02-22T21:55:25.994 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16700 IOs, 6906608640 bytes 2026-02-22T21:55:26.065 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16800 IOs, 6975734784 bytes 2026-02-22T21:55:26.123 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16900 IOs, 7015353344 bytes 2026-02-22T21:55:26.174 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17000 IOs, 7053297664 bytes 2026-02-22T21:55:26.222 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17100 IOs, 7081859072 bytes 2026-02-22T21:55:26.276 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17200 IOs, 7125470208 bytes 2026-02-22T21:55:26.308 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17300 IOs, 7137544192 bytes 2026-02-22T21:55:26.356 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17400 IOs, 7175798784 bytes 2026-02-22T21:55:26.396 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17500 IOs, 7199818752 bytes 2026-02-22T21:55:26.447 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17600 IOs, 7243366400 bytes 2026-02-22T21:55:26.488 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17700 IOs, 7263266816 bytes 2026-02-22T21:55:26.534 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17800 IOs, 7297875968 bytes 2026-02-22T21:55:26.619 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17900 IOs, 7411917824 bytes 2026-02-22T21:55:26.687 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18000 IOs, 7468908544 bytes 2026-02-22T21:55:26.736 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18100 IOs, 7500753920 bytes 2026-02-22T21:55:26.780 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18200 IOs, 7534667776 bytes 2026-02-22T21:55:26.827 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18300 IOs, 7591461888 bytes 2026-02-22T21:55:26.894 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18400 IOs, 7636966400 bytes 2026-02-22T21:55:26.940 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18500 IOs, 7659687936 bytes 2026-02-22T21:55:26.977 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18600 IOs, 7676786688 bytes 2026-02-22T21:55:27.024 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18700 IOs, 7718188032 bytes 2026-02-22T21:55:27.073 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18800 IOs, 7741028352 bytes 2026-02-22T21:55:27.107 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18900 IOs, 7750855680 bytes 2026-02-22T21:55:27.153 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19000 IOs, 7800091648 bytes 2026-02-22T21:55:27.222 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19100 IOs, 7854770176 bytes 2026-02-22T21:55:27.269 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19200 IOs, 7890022400 bytes 2026-02-22T21:55:27.303 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19300 IOs, 7925146624 bytes 2026-02-22T21:55:27.356 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19400 IOs, 7950940160 bytes 2026-02-22T21:55:27.606 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19500 IOs, 7976118272 bytes 2026-02-22T21:55:27.639 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19600 IOs, 7991019520 bytes 2026-02-22T21:55:27.668 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19700 IOs, 7991621632 bytes 2026-02-22T21:55:27.694 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19800 IOs, 7994358784 bytes 2026-02-22T21:55:27.777 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19900 IOs, 8079379456 bytes 2026-02-22T21:55:27.819 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20000 IOs, 8101341184 bytes 2026-02-22T21:55:27.890 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20100 IOs, 8192624640 bytes 2026-02-22T21:55:27.966 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20200 IOs, 8261134336 bytes 2026-02-22T21:55:28.029 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20300 IOs, 8347319296 bytes 2026-02-22T21:55:28.093 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20400 IOs, 8389218304 bytes 2026-02-22T21:55:28.140 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20500 IOs, 8422373376 bytes 2026-02-22T21:55:28.203 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20600 IOs, 8483646464 bytes 2026-02-22T21:55:28.248 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20700 IOs, 8530278400 bytes 2026-02-22T21:55:28.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20800 IOs, 8531384320 bytes 2026-02-22T21:55:28.341 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20900 IOs, 8593591296 bytes 2026-02-22T21:55:28.386 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21000 IOs, 8619275264 bytes 2026-02-22T21:55:28.447 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21100 IOs, 8654404608 bytes 2026-02-22T21:55:28.491 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21200 IOs, 8683069440 bytes 2026-02-22T21:55:28.519 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21300 IOs, 8686451712 bytes 2026-02-22T21:55:28.564 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21400 IOs, 8723728384 bytes 2026-02-22T21:55:28.592 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21500 IOs, 8724345856 bytes 2026-02-22T21:55:28.661 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21600 IOs, 8790299648 bytes 2026-02-22T21:55:28.691 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21700 IOs, 8791733248 bytes 2026-02-22T21:55:28.762 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21800 IOs, 8852726784 bytes 2026-02-22T21:55:28.813 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21900 IOs, 8887506944 bytes 2026-02-22T21:55:28.889 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22000 IOs, 8960657408 bytes 2026-02-22T21:55:28.943 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22100 IOs, 8992979968 bytes 2026-02-22T21:55:28.983 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22200 IOs, 9016468480 bytes 2026-02-22T21:55:29.011 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22300 IOs, 9017098240 bytes 2026-02-22T21:55:29.044 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22400 IOs, 9051491328 bytes 2026-02-22T21:55:29.117 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22500 IOs, 9087415296 bytes 2026-02-22T21:55:29.158 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22600 IOs, 9106541568 bytes 2026-02-22T21:55:29.230 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22700 IOs, 9206360064 bytes 2026-02-22T21:55:29.309 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22800 IOs, 9298813952 bytes 2026-02-22T21:55:29.372 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22900 IOs, 9349816320 bytes 2026-02-22T21:55:29.438 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23000 IOs, 9399634944 bytes 2026-02-22T21:55:29.533 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23100 IOs, 9502099456 bytes 2026-02-22T21:55:29.607 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23200 IOs, 9595358208 bytes 2026-02-22T21:55:29.644 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23300 IOs, 9597906944 bytes 2026-02-22T21:55:29.695 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23400 IOs, 9642488832 bytes 2026-02-22T21:55:29.749 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23500 IOs, 9674382336 bytes 2026-02-22T21:55:29.788 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23600 IOs, 9695300608 bytes 2026-02-22T21:55:29.858 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23700 IOs, 9761565696 bytes 2026-02-22T21:55:29.923 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23800 IOs, 9825408000 bytes 2026-02-22T21:55:29.952 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23900 IOs, 9830977536 bytes 2026-02-22T21:55:29.976 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24000 IOs, 9861288960 bytes 2026-02-22T21:55:30.050 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24100 IOs, 9915196416 bytes 2026-02-22T21:55:30.094 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24200 IOs, 9952906240 bytes 2026-02-22T21:55:30.144 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24300 IOs, 9975830528 bytes 2026-02-22T21:55:30.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ error_code=124 2026-02-22T21:55:30.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set -e 2026-02-22T21:55:30.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' 124 -eq 124 ']' 2026-02-22T21:55:30.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:55:30.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T21:55:30.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_46 2026-02-22T21:55:30.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image_and_enable_mirror cluster2 mirror image_46 snapshot 128M 2026-02-22T21:55:30.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:55:30.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:55:30.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:55:30.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:55:30.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_46 2026-02-22T21:55:30.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:55:30.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T21:55:30.183 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n snapshot ']' 2026-02-22T21:55:30.183 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:55:30.183 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image cluster2 mirror image_46 128M 2026-02-22T21:55:30.183 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:55:30.183 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:55:30.183 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:55:30.183 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:55:30.183 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_46 2026-02-22T21:55:30.183 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:55:30.183 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local size=128 2026-02-22T21:55:30.183 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n 128M ']' 2026-02-22T21:55:30.183 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ size=128M 2026-02-22T21:55:30.184 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:55:30.184 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_46 2026-02-22T21:55:30.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T21:55:30.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:55:30.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' image = image ']' 2026-02-22T21:55:30.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:55:30.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ enable_mirror cluster2 mirror image_46 snapshot 2026-02-22T21:55:30.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:55:30.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:55:30.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_46 2026-02-22T21:55:30.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T21:55:30.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_46 snapshot 2026-02-22T21:55:31.150 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Mirroring enabled 2026-02-22T21:55:31.155 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 info mirror/image_46 2026-02-22T21:55:31.180 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_46': 2026-02-22T21:55:31.180 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 128 MiB in 32 objects 2026-02-22T21:55:31.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T21:55:31.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 1 2026-02-22T21:55:31.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 147a3e3a0eaa 2026-02-22T21:55:31.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.147a3e3a0eaa 2026-02-22T21:55:31.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T21:55:31.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock 2026-02-22T21:55:31.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T21:55:31.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T21:55:31.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:55:30 2026 2026-02-22T21:55:31.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:55:30 2026 2026-02-22T21:55:31.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:55:30 2026 2026-02-22T21:55:31.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T21:55:31.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T21:55:31.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: 04845371-74eb-4487-a2bf-eec2351b191c 2026-02-22T21:55:31.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: true 2026-02-22T21:55:31.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n '' ']' 2026-02-22T21:55:31.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ stress_write_image cluster2 mirror image_46 2026-02-22T21:55:31.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:55:31.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:55:31.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_46 2026-02-22T21:55:31.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-22T21:55:31.184 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local duration=10 2026-02-22T21:55:31.184 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set +e 2026-02-22T21:55:31.184 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ timeout 10s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_46 --debug-rbd=20 --debug-journaler=20 2026-02-22T21:55:31.265 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:100 IOs, 31494144 bytes 2026-02-22T21:55:31.322 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:200 IOs, 77203456 bytes 2026-02-22T21:55:31.375 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:300 IOs, 120126464 bytes 2026-02-22T21:55:31.425 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:400 IOs, 159796224 bytes 2026-02-22T21:55:31.483 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:500 IOs, 211923968 bytes 2026-02-22T21:55:31.558 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:600 IOs, 290356224 bytes 2026-02-22T21:55:31.623 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:700 IOs, 366693376 bytes 2026-02-22T21:55:31.682 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:800 IOs, 415063040 bytes 2026-02-22T21:55:31.755 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:900 IOs, 507629568 bytes 2026-02-22T21:55:31.804 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1000 IOs, 534284288 bytes 2026-02-22T21:55:31.833 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1100 IOs, 534914048 bytes 2026-02-22T21:55:31.896 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1200 IOs, 605869056 bytes 2026-02-22T21:55:31.960 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1300 IOs, 655347712 bytes 2026-02-22T21:55:31.989 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1400 IOs, 655955968 bytes 2026-02-22T21:55:32.027 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1500 IOs, 683587584 bytes 2026-02-22T21:55:32.058 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1600 IOs, 692566016 bytes 2026-02-22T21:55:32.119 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1700 IOs, 748380160 bytes 2026-02-22T21:55:32.174 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1800 IOs, 800163840 bytes 2026-02-22T21:55:32.215 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1900 IOs, 831566848 bytes 2026-02-22T21:55:32.274 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2000 IOs, 869157888 bytes 2026-02-22T21:55:32.332 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2100 IOs, 918412288 bytes 2026-02-22T21:55:32.378 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2200 IOs, 945760256 bytes 2026-02-22T21:55:32.430 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2300 IOs, 988172288 bytes 2026-02-22T21:55:32.471 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2400 IOs, 1006729216 bytes 2026-02-22T21:55:32.504 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2500 IOs, 1017239552 bytes 2026-02-22T21:55:32.544 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2600 IOs, 1039766528 bytes 2026-02-22T21:55:32.587 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2700 IOs, 1068772352 bytes 2026-02-22T21:55:32.637 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2800 IOs, 1114422272 bytes 2026-02-22T21:55:32.698 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2900 IOs, 1173846016 bytes 2026-02-22T21:55:32.779 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3000 IOs, 1271214080 bytes 2026-02-22T21:55:32.810 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3100 IOs, 1271803904 bytes 2026-02-22T21:55:32.862 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3200 IOs, 1321213952 bytes 2026-02-22T21:55:32.902 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3300 IOs, 1348189184 bytes 2026-02-22T21:55:32.940 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3400 IOs, 1369774080 bytes 2026-02-22T21:55:33.003 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3500 IOs, 1434444800 bytes 2026-02-22T21:55:33.070 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3600 IOs, 1518724096 bytes 2026-02-22T21:55:33.103 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3700 IOs, 1528630272 bytes 2026-02-22T21:55:33.159 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3800 IOs, 1585241088 bytes 2026-02-22T21:55:33.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3900 IOs, 1617835008 bytes 2026-02-22T21:55:33.242 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4000 IOs, 1658534912 bytes 2026-02-22T21:55:33.311 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4100 IOs, 1747010560 bytes 2026-02-22T21:55:33.356 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4200 IOs, 1771597824 bytes 2026-02-22T21:55:33.416 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4300 IOs, 1833469952 bytes 2026-02-22T21:55:33.471 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4400 IOs, 1883122688 bytes 2026-02-22T21:55:33.510 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4500 IOs, 1907096576 bytes 2026-02-22T21:55:33.558 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4600 IOs, 1950244864 bytes 2026-02-22T21:55:33.591 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4700 IOs, 1958942720 bytes 2026-02-22T21:55:33.658 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4800 IOs, 2062259200 bytes 2026-02-22T21:55:33.729 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4900 IOs, 2139513856 bytes 2026-02-22T21:55:33.778 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5000 IOs, 2161710080 bytes 2026-02-22T21:55:33.827 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5100 IOs, 2193223680 bytes 2026-02-22T21:55:33.870 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5200 IOs, 2226104320 bytes 2026-02-22T21:55:33.905 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5300 IOs, 2243249152 bytes 2026-02-22T21:55:33.932 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5400 IOs, 2243873792 bytes 2026-02-22T21:55:34.002 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5500 IOs, 2324779008 bytes 2026-02-22T21:55:34.034 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5600 IOs, 2342007808 bytes 2026-02-22T21:55:34.072 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5700 IOs, 2359654400 bytes 2026-02-22T21:55:34.121 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5800 IOs, 2398485504 bytes 2026-02-22T21:55:34.180 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5900 IOs, 2458654720 bytes 2026-02-22T21:55:34.234 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6000 IOs, 2513427456 bytes 2026-02-22T21:55:34.277 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6100 IOs, 2546468864 bytes 2026-02-22T21:55:34.350 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6200 IOs, 2635405312 bytes 2026-02-22T21:55:34.410 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6300 IOs, 2697282560 bytes 2026-02-22T21:55:34.464 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6400 IOs, 2758184960 bytes 2026-02-22T21:55:34.496 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6500 IOs, 2764381184 bytes 2026-02-22T21:55:34.550 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6600 IOs, 2818014208 bytes 2026-02-22T21:55:34.630 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6700 IOs, 2926667776 bytes 2026-02-22T21:55:34.680 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6800 IOs, 2960091136 bytes 2026-02-22T21:55:34.727 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6900 IOs, 3006533632 bytes 2026-02-22T21:55:34.765 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7000 IOs, 3033532416 bytes 2026-02-22T21:55:34.821 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7100 IOs, 3077051392 bytes 2026-02-22T21:55:34.874 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7200 IOs, 3117120512 bytes 2026-02-22T21:55:34.927 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7300 IOs, 3175748608 bytes 2026-02-22T21:55:34.976 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7400 IOs, 3226194944 bytes 2026-02-22T21:55:35.015 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7500 IOs, 3251873792 bytes 2026-02-22T21:55:35.056 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7600 IOs, 3286635520 bytes 2026-02-22T21:55:35.094 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7700 IOs, 3308178432 bytes 2026-02-22T21:55:35.139 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7800 IOs, 3350578176 bytes 2026-02-22T21:55:35.188 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7900 IOs, 3374169088 bytes 2026-02-22T21:55:35.215 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8000 IOs, 3374786560 bytes 2026-02-22T21:55:35.275 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8100 IOs, 3456820224 bytes 2026-02-22T21:55:35.334 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8200 IOs, 3502993408 bytes 2026-02-22T21:55:35.397 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8300 IOs, 3573398528 bytes 2026-02-22T21:55:35.433 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8400 IOs, 3582572544 bytes 2026-02-22T21:55:35.491 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8500 IOs, 3637658624 bytes 2026-02-22T21:55:35.552 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8600 IOs, 3721425920 bytes 2026-02-22T21:55:35.600 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8700 IOs, 3750648832 bytes 2026-02-22T21:55:35.653 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8800 IOs, 3809841152 bytes 2026-02-22T21:55:35.697 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8900 IOs, 3837974528 bytes 2026-02-22T21:55:35.757 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9000 IOs, 3925587968 bytes 2026-02-22T21:55:35.812 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9100 IOs, 3978527744 bytes 2026-02-22T21:55:35.894 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9200 IOs, 4082708480 bytes 2026-02-22T21:55:35.932 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9300 IOs, 4110042112 bytes 2026-02-22T21:55:35.978 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9400 IOs, 4153240576 bytes 2026-02-22T21:55:36.015 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9500 IOs, 4171220992 bytes 2026-02-22T21:55:36.073 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9600 IOs, 4226529280 bytes 2026-02-22T21:55:36.109 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9700 IOs, 4253292544 bytes 2026-02-22T21:55:36.161 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9800 IOs, 4282317824 bytes 2026-02-22T21:55:36.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9900 IOs, 4329360384 bytes 2026-02-22T21:55:36.242 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10000 IOs, 4348338176 bytes 2026-02-22T21:55:36.311 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10100 IOs, 4443622400 bytes 2026-02-22T21:55:36.362 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10200 IOs, 4495644672 bytes 2026-02-22T21:55:36.591 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10300 IOs, 4496270336 bytes 2026-02-22T21:55:36.642 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10400 IOs, 4545234944 bytes 2026-02-22T21:55:36.684 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10500 IOs, 4576081920 bytes 2026-02-22T21:55:36.734 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10600 IOs, 4633216000 bytes 2026-02-22T21:55:36.781 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10700 IOs, 4676784128 bytes 2026-02-22T21:55:36.849 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10800 IOs, 4731536384 bytes 2026-02-22T21:55:36.917 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10900 IOs, 4817842176 bytes 2026-02-22T21:55:36.962 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11000 IOs, 4852359168 bytes 2026-02-22T21:55:36.988 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11100 IOs, 4852958208 bytes 2026-02-22T21:55:37.031 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11200 IOs, 4892197888 bytes 2026-02-22T21:55:37.064 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11300 IOs, 4906178560 bytes 2026-02-22T21:55:37.144 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11400 IOs, 5023031296 bytes 2026-02-22T21:55:37.214 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11500 IOs, 5100132352 bytes 2026-02-22T21:55:37.275 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11600 IOs, 5166574592 bytes 2026-02-22T21:55:37.324 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11700 IOs, 5216673792 bytes 2026-02-22T21:55:37.370 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11800 IOs, 5259876352 bytes 2026-02-22T21:55:37.412 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11900 IOs, 5285709824 bytes 2026-02-22T21:55:37.458 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12000 IOs, 5323570176 bytes 2026-02-22T21:55:37.499 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12100 IOs, 5352484864 bytes 2026-02-22T21:55:37.542 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12200 IOs, 5378230272 bytes 2026-02-22T21:55:37.598 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12300 IOs, 5431564288 bytes 2026-02-22T21:55:37.637 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12400 IOs, 5456515072 bytes 2026-02-22T21:55:37.698 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12500 IOs, 5536514048 bytes 2026-02-22T21:55:37.776 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12600 IOs, 5631797248 bytes 2026-02-22T21:55:37.829 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12700 IOs, 5681427456 bytes 2026-02-22T21:55:37.888 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12800 IOs, 5748064256 bytes 2026-02-22T21:55:37.935 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12900 IOs, 5795487744 bytes 2026-02-22T21:55:37.977 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13000 IOs, 5820951552 bytes 2026-02-22T21:55:38.017 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13100 IOs, 5854125056 bytes 2026-02-22T21:55:38.075 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13200 IOs, 5933002752 bytes 2026-02-22T21:55:38.122 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13300 IOs, 5961654272 bytes 2026-02-22T21:55:38.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13400 IOs, 5991286784 bytes 2026-02-22T21:55:38.215 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13500 IOs, 6029227008 bytes 2026-02-22T21:55:38.251 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13600 IOs, 6055909376 bytes 2026-02-22T21:55:38.295 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13700 IOs, 6081064960 bytes 2026-02-22T21:55:38.352 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13800 IOs, 6126893056 bytes 2026-02-22T21:55:38.387 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13900 IOs, 6143137792 bytes 2026-02-22T21:55:38.432 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14000 IOs, 6181538816 bytes 2026-02-22T21:55:38.461 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14100 IOs, 6182624256 bytes 2026-02-22T21:55:38.513 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14200 IOs, 6234598400 bytes 2026-02-22T21:55:38.553 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14300 IOs, 6267507712 bytes 2026-02-22T21:55:38.583 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14400 IOs, 6284866560 bytes 2026-02-22T21:55:38.637 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14500 IOs, 6323096576 bytes 2026-02-22T21:55:38.687 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14600 IOs, 6379062272 bytes 2026-02-22T21:55:38.737 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14700 IOs, 6409011200 bytes 2026-02-22T21:55:38.774 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14800 IOs, 6433027072 bytes 2026-02-22T21:55:38.827 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14900 IOs, 6483939328 bytes 2026-02-22T21:55:38.875 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15000 IOs, 6530655232 bytes 2026-02-22T21:55:38.939 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15100 IOs, 6607623168 bytes 2026-02-22T21:55:38.983 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15200 IOs, 6642525184 bytes 2026-02-22T21:55:39.046 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15300 IOs, 6715057152 bytes 2026-02-22T21:55:39.089 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15400 IOs, 6756781056 bytes 2026-02-22T21:55:39.152 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15500 IOs, 6817079296 bytes 2026-02-22T21:55:39.203 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15600 IOs, 6863859712 bytes 2026-02-22T21:55:39.236 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15700 IOs, 6876152832 bytes 2026-02-22T21:55:39.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15800 IOs, 6895589376 bytes 2026-02-22T21:55:39.332 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15900 IOs, 6981516288 bytes 2026-02-22T21:55:39.370 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16000 IOs, 6997747712 bytes 2026-02-22T21:55:39.418 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16100 IOs, 7044744192 bytes 2026-02-22T21:55:39.463 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16200 IOs, 7091175424 bytes 2026-02-22T21:55:39.513 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16300 IOs, 7141304320 bytes 2026-02-22T21:55:39.572 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16400 IOs, 7222827008 bytes 2026-02-22T21:55:39.602 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16500 IOs, 7223432192 bytes 2026-02-22T21:55:39.636 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16600 IOs, 7239150592 bytes 2026-02-22T21:55:39.699 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16700 IOs, 7321654272 bytes 2026-02-22T21:55:39.744 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16800 IOs, 7355119616 bytes 2026-02-22T21:55:39.781 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16900 IOs, 7380319232 bytes 2026-02-22T21:55:39.875 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17000 IOs, 7507002368 bytes 2026-02-22T21:55:39.937 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17100 IOs, 7583618048 bytes 2026-02-22T21:55:39.973 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17200 IOs, 7603323904 bytes 2026-02-22T21:55:40.020 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17300 IOs, 7639499776 bytes 2026-02-22T21:55:40.065 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17400 IOs, 7683011584 bytes 2026-02-22T21:55:40.110 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17500 IOs, 7742773248 bytes 2026-02-22T21:55:40.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17600 IOs, 7788634112 bytes 2026-02-22T21:55:40.241 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17700 IOs, 7867521024 bytes 2026-02-22T21:55:40.288 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17800 IOs, 7910581248 bytes 2026-02-22T21:55:40.325 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17900 IOs, 7932769280 bytes 2026-02-22T21:55:40.355 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18000 IOs, 7945783296 bytes 2026-02-22T21:55:40.404 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18100 IOs, 8009758720 bytes 2026-02-22T21:55:40.485 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18200 IOs, 8105350144 bytes 2026-02-22T21:55:40.524 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18300 IOs, 8131684352 bytes 2026-02-22T21:55:40.583 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18400 IOs, 8207466496 bytes 2026-02-22T21:55:40.667 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18500 IOs, 8328568832 bytes 2026-02-22T21:55:40.702 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18600 IOs, 8344823808 bytes 2026-02-22T21:55:40.743 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18700 IOs, 8379983872 bytes 2026-02-22T21:55:40.796 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18800 IOs, 8438461440 bytes 2026-02-22T21:55:40.833 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18900 IOs, 8458856448 bytes 2026-02-22T21:55:40.886 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19000 IOs, 8495127552 bytes 2026-02-22T21:55:40.933 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19100 IOs, 8542111744 bytes 2026-02-22T21:55:40.983 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19200 IOs, 8587463680 bytes 2026-02-22T21:55:41.020 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19300 IOs, 8623614976 bytes 2026-02-22T21:55:41.063 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19400 IOs, 8649270272 bytes 2026-02-22T21:55:41.110 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19500 IOs, 8694685696 bytes 2026-02-22T21:55:41.153 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19600 IOs, 8724090880 bytes 2026-02-22T21:55:41.185 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19700 IOs, 8724724736 bytes 2026-02-22T21:55:41.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ error_code=124 2026-02-22T21:55:41.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set -e 2026-02-22T21:55:41.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' 124 -eq 124 ']' 2026-02-22T21:55:41.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:55:41.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T21:55:41.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_47 2026-02-22T21:55:41.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image_and_enable_mirror cluster2 mirror image_47 snapshot 128M 2026-02-22T21:55:41.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:55:41.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:55:41.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:55:41.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:55:41.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_47 2026-02-22T21:55:41.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:55:41.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T21:55:41.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n snapshot ']' 2026-02-22T21:55:41.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:55:41.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image cluster2 mirror image_47 128M 2026-02-22T21:55:41.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:55:41.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:55:41.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:55:41.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:55:41.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_47 2026-02-22T21:55:41.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:55:41.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local size=128 2026-02-22T21:55:41.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n 128M ']' 2026-02-22T21:55:41.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ size=128M 2026-02-22T21:55:41.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:55:41.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_47 2026-02-22T21:55:41.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T21:55:41.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:55:41.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' image = image ']' 2026-02-22T21:55:41.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:55:41.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ enable_mirror cluster2 mirror image_47 snapshot 2026-02-22T21:55:41.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:55:41.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:55:41.222 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_47 2026-02-22T21:55:41.222 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T21:55:41.222 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_47 snapshot 2026-02-22T21:55:42.160 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Mirroring enabled 2026-02-22T21:55:42.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 info mirror/image_47 2026-02-22T21:55:42.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_47': 2026-02-22T21:55:42.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 128 MiB in 32 objects 2026-02-22T21:55:42.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T21:55:42.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 1 2026-02-22T21:55:42.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 148271d5878f 2026-02-22T21:55:42.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.148271d5878f 2026-02-22T21:55:42.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T21:55:42.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock 2026-02-22T21:55:42.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T21:55:42.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T21:55:42.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:55:41 2026 2026-02-22T21:55:42.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:55:41 2026 2026-02-22T21:55:42.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:55:41 2026 2026-02-22T21:55:42.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T21:55:42.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T21:55:42.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: 1c0dd8a5-46f9-44d8-9de4-48e37b2678f7 2026-02-22T21:55:42.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: true 2026-02-22T21:55:42.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n '' ']' 2026-02-22T21:55:42.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ stress_write_image cluster2 mirror image_47 2026-02-22T21:55:42.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:55:42.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:55:42.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_47 2026-02-22T21:55:42.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-22T21:55:42.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local duration=10 2026-02-22T21:55:42.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set +e 2026-02-22T21:55:42.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ timeout 10s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_47 --debug-rbd=20 --debug-journaler=20 2026-02-22T21:55:42.288 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:100 IOs, 70900736 bytes 2026-02-22T21:55:42.338 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:200 IOs, 120589312 bytes 2026-02-22T21:55:42.382 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:300 IOs, 137705472 bytes 2026-02-22T21:55:42.430 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:400 IOs, 179044352 bytes 2026-02-22T21:55:42.485 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:500 IOs, 229787648 bytes 2026-02-22T21:55:42.519 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:600 IOs, 239491072 bytes 2026-02-22T21:55:42.558 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:700 IOs, 264912896 bytes 2026-02-22T21:55:42.586 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:800 IOs, 265524224 bytes 2026-02-22T21:55:42.635 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:900 IOs, 305699840 bytes 2026-02-22T21:55:42.697 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1000 IOs, 366115840 bytes 2026-02-22T21:55:42.740 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1100 IOs, 426180608 bytes 2026-02-22T21:55:42.807 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1200 IOs, 491064320 bytes 2026-02-22T21:55:42.837 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1300 IOs, 492517376 bytes 2026-02-22T21:55:42.906 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1400 IOs, 578999296 bytes 2026-02-22T21:55:42.956 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1500 IOs, 612123648 bytes 2026-02-22T21:55:42.990 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1600 IOs, 626963456 bytes 2026-02-22T21:55:43.028 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1700 IOs, 650373120 bytes 2026-02-22T21:55:43.097 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1800 IOs, 714281984 bytes 2026-02-22T21:55:43.147 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1900 IOs, 758649856 bytes 2026-02-22T21:55:43.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2000 IOs, 777031680 bytes 2026-02-22T21:55:43.244 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2100 IOs, 824344576 bytes 2026-02-22T21:55:43.286 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2200 IOs, 849256448 bytes 2026-02-22T21:55:43.327 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2300 IOs, 891848704 bytes 2026-02-22T21:55:43.381 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2400 IOs, 925722624 bytes 2026-02-22T21:55:43.426 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2500 IOs, 971145216 bytes 2026-02-22T21:55:43.470 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2600 IOs, 990447616 bytes 2026-02-22T21:55:43.524 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2700 IOs, 1036279808 bytes 2026-02-22T21:55:43.583 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2800 IOs, 1093650432 bytes 2026-02-22T21:55:43.617 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2900 IOs, 1106269184 bytes 2026-02-22T21:55:43.652 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3000 IOs, 1116499968 bytes 2026-02-22T21:55:43.706 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3100 IOs, 1172909056 bytes 2026-02-22T21:55:43.756 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3200 IOs, 1225722880 bytes 2026-02-22T21:55:43.793 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3300 IOs, 1242624000 bytes 2026-02-22T21:55:43.829 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3400 IOs, 1277891584 bytes 2026-02-22T21:55:43.887 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3500 IOs, 1330003968 bytes 2026-02-22T21:55:43.965 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3600 IOs, 1396817920 bytes 2026-02-22T21:55:44.033 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3700 IOs, 1483070464 bytes 2026-02-22T21:55:44.086 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3800 IOs, 1520816128 bytes 2026-02-22T21:55:44.130 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3900 IOs, 1547082752 bytes 2026-02-22T21:55:44.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4000 IOs, 1570717696 bytes 2026-02-22T21:55:44.215 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4100 IOs, 1571302400 bytes 2026-02-22T21:55:44.276 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4200 IOs, 1626739712 bytes 2026-02-22T21:55:44.349 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4300 IOs, 1708194816 bytes 2026-02-22T21:55:44.395 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4400 IOs, 1748401152 bytes 2026-02-22T21:55:44.456 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4500 IOs, 1807681536 bytes 2026-02-22T21:55:44.505 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4600 IOs, 1845255168 bytes 2026-02-22T21:55:44.556 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4700 IOs, 1880935424 bytes 2026-02-22T21:55:44.623 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4800 IOs, 1951156224 bytes 2026-02-22T21:55:44.677 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4900 IOs, 1999763456 bytes 2026-02-22T21:55:44.720 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5000 IOs, 2030415872 bytes 2026-02-22T21:55:44.775 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5100 IOs, 2082834432 bytes 2026-02-22T21:55:44.838 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5200 IOs, 2156845056 bytes 2026-02-22T21:55:44.873 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5300 IOs, 2162766848 bytes 2026-02-22T21:55:44.937 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5400 IOs, 2234061824 bytes 2026-02-22T21:55:44.977 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5500 IOs, 2261601280 bytes 2026-02-22T21:55:45.010 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5600 IOs, 2264433664 bytes 2026-02-22T21:55:45.051 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5700 IOs, 2286763008 bytes 2026-02-22T21:55:45.108 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5800 IOs, 2342816768 bytes 2026-02-22T21:55:45.156 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5900 IOs, 2376769536 bytes 2026-02-22T21:55:45.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6000 IOs, 2401653760 bytes 2026-02-22T21:55:45.254 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6100 IOs, 2427575296 bytes 2026-02-22T21:55:45.309 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6200 IOs, 2478863360 bytes 2026-02-22T21:55:45.351 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6300 IOs, 2502085632 bytes 2026-02-22T21:55:45.384 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6400 IOs, 2523157504 bytes 2026-02-22T21:55:45.441 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6500 IOs, 2563024896 bytes 2026-02-22T21:55:45.475 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6600 IOs, 2581305344 bytes 2026-02-22T21:55:45.530 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6700 IOs, 2622399488 bytes 2026-02-22T21:55:45.568 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6800 IOs, 2648475648 bytes 2026-02-22T21:55:45.607 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6900 IOs, 2673503232 bytes 2026-02-22T21:55:45.669 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7000 IOs, 2739795968 bytes 2026-02-22T21:55:45.717 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7100 IOs, 2795853824 bytes 2026-02-22T21:55:45.770 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7200 IOs, 2832818176 bytes 2026-02-22T21:55:45.829 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7300 IOs, 2916446208 bytes 2026-02-22T21:55:45.885 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7400 IOs, 2943082496 bytes 2026-02-22T21:55:45.936 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7500 IOs, 3008726016 bytes 2026-02-22T21:55:46.008 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7600 IOs, 3068632064 bytes 2026-02-22T21:55:46.071 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7700 IOs, 3136774144 bytes 2026-02-22T21:55:46.121 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7800 IOs, 3180236800 bytes 2026-02-22T21:55:46.170 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7900 IOs, 3217084416 bytes 2026-02-22T21:55:46.218 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8000 IOs, 3258580992 bytes 2026-02-22T21:55:46.302 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8100 IOs, 3345377280 bytes 2026-02-22T21:55:46.347 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8200 IOs, 3369967616 bytes 2026-02-22T21:55:46.386 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8300 IOs, 3389067264 bytes 2026-02-22T21:55:46.432 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8400 IOs, 3419576320 bytes 2026-02-22T21:55:46.468 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8500 IOs, 3444336640 bytes 2026-02-22T21:55:46.557 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8600 IOs, 3576868864 bytes 2026-02-22T21:55:46.600 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8700 IOs, 3588710400 bytes 2026-02-22T21:55:46.643 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8800 IOs, 3615080448 bytes 2026-02-22T21:55:46.693 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8900 IOs, 3662187520 bytes 2026-02-22T21:55:46.758 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9000 IOs, 3735282688 bytes 2026-02-22T21:55:46.809 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9100 IOs, 3788246016 bytes 2026-02-22T21:55:46.861 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9200 IOs, 3831505920 bytes 2026-02-22T21:55:46.906 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9300 IOs, 3870746624 bytes 2026-02-22T21:55:46.957 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9400 IOs, 3918694400 bytes 2026-02-22T21:55:47.014 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9500 IOs, 3952345088 bytes 2026-02-22T21:55:47.051 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9600 IOs, 3970097152 bytes 2026-02-22T21:55:47.110 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9700 IOs, 4025250816 bytes 2026-02-22T21:55:47.159 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9800 IOs, 4061113344 bytes 2026-02-22T21:55:47.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9900 IOs, 4076192768 bytes 2026-02-22T21:55:47.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10000 IOs, 4162382848 bytes 2026-02-22T21:55:47.301 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10100 IOs, 4162990080 bytes 2026-02-22T21:55:47.368 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10200 IOs, 4248908800 bytes 2026-02-22T21:55:47.446 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10300 IOs, 4325198848 bytes 2026-02-22T21:55:47.488 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10400 IOs, 4370633728 bytes 2026-02-22T21:55:47.543 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10500 IOs, 4398614528 bytes 2026-02-22T21:55:47.615 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10600 IOs, 4485350400 bytes 2026-02-22T21:55:47.659 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10700 IOs, 4493261824 bytes 2026-02-22T21:55:47.702 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10800 IOs, 4524568576 bytes 2026-02-22T21:55:47.750 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10900 IOs, 4569379840 bytes 2026-02-22T21:55:47.812 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11000 IOs, 4655872000 bytes 2026-02-22T21:55:47.860 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11100 IOs, 4692302848 bytes 2026-02-22T21:55:47.916 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11200 IOs, 4751067136 bytes 2026-02-22T21:55:47.981 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11300 IOs, 4834254848 bytes 2026-02-22T21:55:48.043 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11400 IOs, 4907851776 bytes 2026-02-22T21:55:48.089 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11500 IOs, 4949152768 bytes 2026-02-22T21:55:48.124 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11600 IOs, 4963305472 bytes 2026-02-22T21:55:48.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11700 IOs, 5030802432 bytes 2026-02-22T21:55:48.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11800 IOs, 5117366272 bytes 2026-02-22T21:55:48.295 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11900 IOs, 5137344512 bytes 2026-02-22T21:55:48.323 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12000 IOs, 5137955840 bytes 2026-02-22T21:55:48.364 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12100 IOs, 5160080384 bytes 2026-02-22T21:55:48.422 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12200 IOs, 5221206016 bytes 2026-02-22T21:55:48.481 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12300 IOs, 5278678016 bytes 2026-02-22T21:55:48.543 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12400 IOs, 5341384704 bytes 2026-02-22T21:55:48.610 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12500 IOs, 5409168384 bytes 2026-02-22T21:55:48.654 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12600 IOs, 5437921280 bytes 2026-02-22T21:55:48.689 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12700 IOs, 5455787008 bytes 2026-02-22T21:55:48.730 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12800 IOs, 5479543808 bytes 2026-02-22T21:55:48.778 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12900 IOs, 5513640960 bytes 2026-02-22T21:55:48.826 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13000 IOs, 5551234048 bytes 2026-02-22T21:55:48.867 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13100 IOs, 5589597184 bytes 2026-02-22T21:55:48.930 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13200 IOs, 5641400320 bytes 2026-02-22T21:55:48.978 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13300 IOs, 5677117440 bytes 2026-02-22T21:55:49.015 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13400 IOs, 5701560320 bytes 2026-02-22T21:55:49.047 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13500 IOs, 5702157312 bytes 2026-02-22T21:55:49.120 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13600 IOs, 5780151296 bytes 2026-02-22T21:55:49.160 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13700 IOs, 5811052544 bytes 2026-02-22T21:55:49.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13800 IOs, 5878871040 bytes 2026-02-22T21:55:49.282 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13900 IOs, 5930765312 bytes 2026-02-22T21:55:49.322 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14000 IOs, 5941907456 bytes 2026-02-22T21:55:49.381 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14100 IOs, 6003678208 bytes 2026-02-22T21:55:49.440 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14200 IOs, 6072425472 bytes 2026-02-22T21:55:49.509 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14300 IOs, 6142740480 bytes 2026-02-22T21:55:49.577 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14400 IOs, 6227992576 bytes 2026-02-22T21:55:49.635 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14500 IOs, 6291999744 bytes 2026-02-22T21:55:49.672 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14600 IOs, 6309669888 bytes 2026-02-22T21:55:49.713 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14700 IOs, 6332299264 bytes 2026-02-22T21:55:49.746 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14800 IOs, 6340202496 bytes 2026-02-22T21:55:49.804 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14900 IOs, 6394840064 bytes 2026-02-22T21:55:49.849 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15000 IOs, 6424568832 bytes 2026-02-22T21:55:49.908 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15100 IOs, 6480002048 bytes 2026-02-22T21:55:49.962 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15200 IOs, 6524910592 bytes 2026-02-22T21:55:49.996 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15300 IOs, 6537066496 bytes 2026-02-22T21:55:50.023 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15400 IOs, 6537688064 bytes 2026-02-22T21:55:50.058 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15500 IOs, 6566756352 bytes 2026-02-22T21:55:50.109 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15600 IOs, 6606000128 bytes 2026-02-22T21:55:50.141 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15700 IOs, 6606627840 bytes 2026-02-22T21:55:50.184 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15800 IOs, 6633993216 bytes 2026-02-22T21:55:50.213 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15900 IOs, 6637473792 bytes 2026-02-22T21:55:50.265 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16000 IOs, 6672167936 bytes 2026-02-22T21:55:50.305 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16100 IOs, 6706169856 bytes 2026-02-22T21:55:50.352 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16200 IOs, 6739708928 bytes 2026-02-22T21:55:50.421 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16300 IOs, 6828216320 bytes 2026-02-22T21:55:50.465 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16400 IOs, 6853601280 bytes 2026-02-22T21:55:50.506 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16500 IOs, 6878479360 bytes 2026-02-22T21:55:50.559 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16600 IOs, 6925762560 bytes 2026-02-22T21:55:50.608 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16700 IOs, 6967465984 bytes 2026-02-22T21:55:50.654 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16800 IOs, 7001294848 bytes 2026-02-22T21:55:50.697 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16900 IOs, 7032195072 bytes 2026-02-22T21:55:50.747 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17000 IOs, 7078210560 bytes 2026-02-22T21:55:50.812 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17100 IOs, 7159434240 bytes 2026-02-22T21:55:50.855 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17200 IOs, 7184328704 bytes 2026-02-22T21:55:50.883 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17300 IOs, 7188978688 bytes 2026-02-22T21:55:50.928 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17400 IOs, 7211643904 bytes 2026-02-22T21:55:50.982 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17500 IOs, 7265378304 bytes 2026-02-22T21:55:51.012 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17600 IOs, 7265989632 bytes 2026-02-22T21:55:51.062 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17700 IOs, 7305062400 bytes 2026-02-22T21:55:51.091 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17800 IOs, 7305670656 bytes 2026-02-22T21:55:51.136 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17900 IOs, 7337719808 bytes 2026-02-22T21:55:51.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18000 IOs, 7428232192 bytes 2026-02-22T21:55:51.238 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18100 IOs, 7428835328 bytes 2026-02-22T21:55:51.302 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18200 IOs, 7494574080 bytes 2026-02-22T21:55:51.353 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18300 IOs, 7511300096 bytes 2026-02-22T21:55:51.397 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18400 IOs, 7539426304 bytes 2026-02-22T21:55:51.474 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18500 IOs, 7636602880 bytes 2026-02-22T21:55:51.537 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18600 IOs, 7678691328 bytes 2026-02-22T21:55:51.594 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18700 IOs, 7728256000 bytes 2026-02-22T21:55:51.642 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18800 IOs, 7762453504 bytes 2026-02-22T21:55:51.672 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18900 IOs, 7764887552 bytes 2026-02-22T21:55:51.705 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19000 IOs, 7776760832 bytes 2026-02-22T21:55:51.735 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19100 IOs, 7781848064 bytes 2026-02-22T21:55:51.808 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19200 IOs, 7871591424 bytes 2026-02-22T21:55:51.862 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19300 IOs, 7918563328 bytes 2026-02-22T21:55:52.101 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19400 IOs, 7937286144 bytes 2026-02-22T21:55:52.142 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19500 IOs, 7950518272 bytes 2026-02-22T21:55:52.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19600 IOs, 7988898816 bytes 2026-02-22T21:55:52.196 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ error_code=124 2026-02-22T21:55:52.196 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set -e 2026-02-22T21:55:52.196 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' 124 -eq 124 ']' 2026-02-22T21:55:52.196 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:55:52.196 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T21:55:52.196 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_48 2026-02-22T21:55:52.196 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image_and_enable_mirror cluster2 mirror image_48 snapshot 128M 2026-02-22T21:55:52.196 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:55:52.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:55:52.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:55:52.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:55:52.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_48 2026-02-22T21:55:52.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:55:52.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T21:55:52.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n snapshot ']' 2026-02-22T21:55:52.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:55:52.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image cluster2 mirror image_48 128M 2026-02-22T21:55:52.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:55:52.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:55:52.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:55:52.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:55:52.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_48 2026-02-22T21:55:52.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:55:52.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local size=128 2026-02-22T21:55:52.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n 128M ']' 2026-02-22T21:55:52.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ size=128M 2026-02-22T21:55:52.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:55:52.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_48 2026-02-22T21:55:52.229 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T21:55:52.229 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:55:52.229 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' image = image ']' 2026-02-22T21:55:52.229 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:55:52.229 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ enable_mirror cluster2 mirror image_48 snapshot 2026-02-22T21:55:52.229 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:55:52.229 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:55:52.229 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_48 2026-02-22T21:55:52.229 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T21:55:52.229 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_48 snapshot 2026-02-22T21:55:53.171 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Mirroring enabled 2026-02-22T21:55:53.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 info mirror/image_48 2026-02-22T21:55:53.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_48': 2026-02-22T21:55:53.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 128 MiB in 32 objects 2026-02-22T21:55:53.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T21:55:53.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 1 2026-02-22T21:55:53.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 148b5202638f 2026-02-22T21:55:53.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.148b5202638f 2026-02-22T21:55:53.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T21:55:53.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock 2026-02-22T21:55:53.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T21:55:53.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T21:55:53.203 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:55:52 2026 2026-02-22T21:55:53.203 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:55:52 2026 2026-02-22T21:55:53.203 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:55:52 2026 2026-02-22T21:55:53.203 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T21:55:53.203 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T21:55:53.203 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: 073c2a09-55c1-4f22-ad52-735988b4ed7b 2026-02-22T21:55:53.203 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: true 2026-02-22T21:55:53.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n '' ']' 2026-02-22T21:55:53.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ stress_write_image cluster2 mirror image_48 2026-02-22T21:55:53.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:55:53.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:55:53.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_48 2026-02-22T21:55:53.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-22T21:55:53.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local duration=14 2026-02-22T21:55:53.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set +e 2026-02-22T21:55:53.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ timeout 14s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_48 --debug-rbd=20 --debug-journaler=20 2026-02-22T21:55:53.275 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:100 IOs, 15388672 bytes 2026-02-22T21:55:53.309 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:200 IOs, 25333760 bytes 2026-02-22T21:55:53.337 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:300 IOs, 29675520 bytes 2026-02-22T21:55:53.386 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:400 IOs, 82978816 bytes 2026-02-22T21:55:53.440 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:500 IOs, 135388160 bytes 2026-02-22T21:55:53.498 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:600 IOs, 185615360 bytes 2026-02-22T21:55:53.536 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:700 IOs, 204038144 bytes 2026-02-22T21:55:53.572 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:800 IOs, 226676736 bytes 2026-02-22T21:55:53.623 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:900 IOs, 279818240 bytes 2026-02-22T21:55:53.674 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1000 IOs, 329182208 bytes 2026-02-22T21:55:53.719 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1100 IOs, 355832832 bytes 2026-02-22T21:55:53.770 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1200 IOs, 408380416 bytes 2026-02-22T21:55:53.803 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1300 IOs, 418308096 bytes 2026-02-22T21:55:53.862 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1400 IOs, 473992192 bytes 2026-02-22T21:55:53.890 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1500 IOs, 474584064 bytes 2026-02-22T21:55:53.946 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1600 IOs, 527461376 bytes 2026-02-22T21:55:53.981 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1700 IOs, 545001472 bytes 2026-02-22T21:55:54.027 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1800 IOs, 581756928 bytes 2026-02-22T21:55:54.069 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1900 IOs, 613374976 bytes 2026-02-22T21:55:54.117 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2000 IOs, 664226816 bytes 2026-02-22T21:55:54.160 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2100 IOs, 687066112 bytes 2026-02-22T21:55:54.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2200 IOs, 707198976 bytes 2026-02-22T21:55:54.264 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2300 IOs, 792994816 bytes 2026-02-22T21:55:54.321 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2400 IOs, 817720320 bytes 2026-02-22T21:55:54.347 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2500 IOs, 818315264 bytes 2026-02-22T21:55:54.424 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2600 IOs, 919999488 bytes 2026-02-22T21:55:54.488 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2700 IOs, 981146624 bytes 2026-02-22T21:55:54.558 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2800 IOs, 1072350208 bytes 2026-02-22T21:55:54.610 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2900 IOs, 1144908800 bytes 2026-02-22T21:55:54.662 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3000 IOs, 1180971008 bytes 2026-02-22T21:55:54.708 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3100 IOs, 1220903936 bytes 2026-02-22T21:55:54.750 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3200 IOs, 1257073664 bytes 2026-02-22T21:55:54.780 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3300 IOs, 1269903360 bytes 2026-02-22T21:55:54.819 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3400 IOs, 1292957696 bytes 2026-02-22T21:55:54.885 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3500 IOs, 1377729536 bytes 2026-02-22T21:55:54.943 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3600 IOs, 1455723520 bytes 2026-02-22T21:55:54.994 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3700 IOs, 1478975488 bytes 2026-02-22T21:55:55.031 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3800 IOs, 1508467712 bytes 2026-02-22T21:55:55.094 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3900 IOs, 1571678208 bytes 2026-02-22T21:55:55.146 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4000 IOs, 1614022656 bytes 2026-02-22T21:55:55.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4100 IOs, 1647674368 bytes 2026-02-22T21:55:55.230 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4200 IOs, 1672469504 bytes 2026-02-22T21:55:55.262 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4300 IOs, 1685560320 bytes 2026-02-22T21:55:55.310 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4400 IOs, 1710186496 bytes 2026-02-22T21:55:55.386 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4500 IOs, 1817447424 bytes 2026-02-22T21:55:55.442 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4600 IOs, 1866591232 bytes 2026-02-22T21:55:55.473 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4700 IOs, 1873948672 bytes 2026-02-22T21:55:55.523 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4800 IOs, 1922495488 bytes 2026-02-22T21:55:55.558 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4900 IOs, 1936192512 bytes 2026-02-22T21:55:55.612 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5000 IOs, 1996314624 bytes 2026-02-22T21:55:55.657 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5100 IOs, 2032827392 bytes 2026-02-22T21:55:55.696 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5200 IOs, 2051690496 bytes 2026-02-22T21:55:55.732 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5300 IOs, 2068428800 bytes 2026-02-22T21:55:55.774 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5400 IOs, 2109388800 bytes 2026-02-22T21:55:55.818 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5500 IOs, 2129251328 bytes 2026-02-22T21:55:55.852 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5600 IOs, 2139246592 bytes 2026-02-22T21:55:55.911 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5700 IOs, 2192526336 bytes 2026-02-22T21:55:55.946 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5800 IOs, 2207249408 bytes 2026-02-22T21:55:55.982 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5900 IOs, 2222441472 bytes 2026-02-22T21:55:56.013 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6000 IOs, 2231325696 bytes 2026-02-22T21:55:56.060 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6100 IOs, 2256406528 bytes 2026-02-22T21:55:56.118 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6200 IOs, 2318807040 bytes 2026-02-22T21:55:56.169 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6300 IOs, 2367066112 bytes 2026-02-22T21:55:56.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6400 IOs, 2385956864 bytes 2026-02-22T21:55:56.231 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6500 IOs, 2388087808 bytes 2026-02-22T21:55:56.281 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6600 IOs, 2454583296 bytes 2026-02-22T21:55:56.353 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6700 IOs, 2531779584 bytes 2026-02-22T21:55:56.404 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6800 IOs, 2577086464 bytes 2026-02-22T21:55:56.448 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6900 IOs, 2614342656 bytes 2026-02-22T21:55:56.480 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7000 IOs, 2629600256 bytes 2026-02-22T21:55:56.530 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7100 IOs, 2669364224 bytes 2026-02-22T21:55:56.571 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7200 IOs, 2695673856 bytes 2026-02-22T21:55:56.641 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7300 IOs, 2789726208 bytes 2026-02-22T21:55:56.690 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7400 IOs, 2828943360 bytes 2026-02-22T21:55:56.744 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7500 IOs, 2881051648 bytes 2026-02-22T21:55:56.789 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7600 IOs, 2909830144 bytes 2026-02-22T21:55:56.851 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7700 IOs, 2977652736 bytes 2026-02-22T21:55:56.888 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7800 IOs, 2997161984 bytes 2026-02-22T21:55:56.927 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7900 IOs, 3019739136 bytes 2026-02-22T21:55:56.985 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8000 IOs, 3083593728 bytes 2026-02-22T21:55:57.025 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8100 IOs, 3104637952 bytes 2026-02-22T21:55:57.099 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8200 IOs, 3214212096 bytes 2026-02-22T21:55:57.130 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8300 IOs, 3214825472 bytes 2026-02-22T21:55:57.156 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8400 IOs, 3215464448 bytes 2026-02-22T21:55:57.227 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8500 IOs, 3316454400 bytes 2026-02-22T21:55:57.276 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8600 IOs, 3354155008 bytes 2026-02-22T21:55:57.299 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.840519751620654) > weight (0.25) 2026-02-22T21:55:57.299 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.1 2026-02-22T21:55:57.300 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Sent signal 15 2026-02-22T21:55:57.300 INFO:tasks.rbd_mirror_thrash:waiting for 7.6844728225001955 secs before reviving daemons 2026-02-22T21:55:57.301 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial026.stderr:2026-02-22T21:55:57.300+0000 7f9ee505b640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 (PID: 56979) UID: 1000 2026-02-22T21:55:57.347 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8700 IOs, 3428385792 bytes 2026-02-22T21:55:57.416 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8800 IOs, 3531150336 bytes 2026-02-22T21:55:57.458 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8900 IOs, 3542562816 bytes 2026-02-22T21:55:57.518 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9000 IOs, 3608761344 bytes 2026-02-22T21:55:57.577 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9100 IOs, 3667745792 bytes 2026-02-22T21:55:57.613 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9200 IOs, 3687187456 bytes 2026-02-22T21:55:57.641 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9300 IOs, 3693957120 bytes 2026-02-22T21:55:57.689 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9400 IOs, 3730627584 bytes 2026-02-22T21:55:57.727 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9500 IOs, 3756223488 bytes 2026-02-22T21:55:57.752 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9600 IOs, 3756934144 bytes 2026-02-22T21:55:57.817 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9700 IOs, 3834208256 bytes 2026-02-22T21:55:57.887 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9800 IOs, 3931489280 bytes 2026-02-22T21:55:57.921 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9900 IOs, 3943569408 bytes 2026-02-22T21:55:57.989 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10000 IOs, 4019378176 bytes 2026-02-22T21:55:58.052 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10100 IOs, 4105202688 bytes 2026-02-22T21:55:58.104 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10200 IOs, 4155432960 bytes 2026-02-22T21:55:58.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10300 IOs, 4236140544 bytes 2026-02-22T21:55:58.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10400 IOs, 4265341952 bytes 2026-02-22T21:55:58.267 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10500 IOs, 4326660096 bytes 2026-02-22T21:55:58.304 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10600 IOs, 4349254656 bytes 2026-02-22T21:55:58.338 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10700 IOs, 4362047488 bytes 2026-02-22T21:55:58.369 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10800 IOs, 4376712192 bytes 2026-02-22T21:55:58.413 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10900 IOs, 4406192128 bytes 2026-02-22T21:55:58.465 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11000 IOs, 4443034624 bytes 2026-02-22T21:55:58.499 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11100 IOs, 4456990720 bytes 2026-02-22T21:55:58.564 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11200 IOs, 4532861952 bytes 2026-02-22T21:55:58.599 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11300 IOs, 4546013184 bytes 2026-02-22T21:55:58.636 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11400 IOs, 4565601280 bytes 2026-02-22T21:55:58.664 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11500 IOs, 4566222848 bytes 2026-02-22T21:55:58.710 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11600 IOs, 4610669568 bytes 2026-02-22T21:55:58.747 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11700 IOs, 4636886016 bytes 2026-02-22T21:55:58.793 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11800 IOs, 4678924288 bytes 2026-02-22T21:55:58.850 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11900 IOs, 4732695552 bytes 2026-02-22T21:55:58.899 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12000 IOs, 4772890624 bytes 2026-02-22T21:55:58.977 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12100 IOs, 4880814080 bytes 2026-02-22T21:55:59.030 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12200 IOs, 4926837760 bytes 2026-02-22T21:55:59.079 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12300 IOs, 4973750272 bytes 2026-02-22T21:55:59.123 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12400 IOs, 5007660032 bytes 2026-02-22T21:55:59.151 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12500 IOs, 5008950272 bytes 2026-02-22T21:55:59.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12600 IOs, 5029325824 bytes 2026-02-22T21:55:59.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12700 IOs, 5064306688 bytes 2026-02-22T21:55:59.261 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12800 IOs, 5077634048 bytes 2026-02-22T21:55:59.319 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12900 IOs, 5127869440 bytes 2026-02-22T21:55:59.370 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13000 IOs, 5158535168 bytes 2026-02-22T21:55:59.405 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13100 IOs, 5169075200 bytes 2026-02-22T21:55:59.448 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13200 IOs, 5202127872 bytes 2026-02-22T21:55:59.490 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13300 IOs, 5247462400 bytes 2026-02-22T21:55:59.531 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13400 IOs, 5264991232 bytes 2026-02-22T21:55:59.572 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13500 IOs, 5286758400 bytes 2026-02-22T21:55:59.600 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13600 IOs, 5289665536 bytes 2026-02-22T21:55:59.650 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13700 IOs, 5344016384 bytes 2026-02-22T21:55:59.687 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13800 IOs, 5363094528 bytes 2026-02-22T21:55:59.742 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13900 IOs, 5417657344 bytes 2026-02-22T21:55:59.808 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14000 IOs, 5497878528 bytes 2026-02-22T21:55:59.854 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14100 IOs, 5527699456 bytes 2026-02-22T21:55:59.885 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14200 IOs, 5535072256 bytes 2026-02-22T21:55:59.932 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14300 IOs, 5570964480 bytes 2026-02-22T21:55:59.976 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14400 IOs, 5597409280 bytes 2026-02-22T21:56:00.012 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14500 IOs, 5611360256 bytes 2026-02-22T21:56:00.058 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14600 IOs, 5652714496 bytes 2026-02-22T21:56:00.097 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14700 IOs, 5678135296 bytes 2026-02-22T21:56:00.154 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14800 IOs, 5735942144 bytes 2026-02-22T21:56:00.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14900 IOs, 5788761088 bytes 2026-02-22T21:56:00.262 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15000 IOs, 5829708800 bytes 2026-02-22T21:56:00.317 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15100 IOs, 5886958592 bytes 2026-02-22T21:56:00.396 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15200 IOs, 5983423488 bytes 2026-02-22T21:56:00.447 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15300 IOs, 6029546496 bytes 2026-02-22T21:56:00.507 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15400 IOs, 6103623680 bytes 2026-02-22T21:56:00.571 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15500 IOs, 6150981632 bytes 2026-02-22T21:56:00.625 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15600 IOs, 6211191808 bytes 2026-02-22T21:56:00.665 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15700 IOs, 6237338624 bytes 2026-02-22T21:56:00.701 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15800 IOs, 6258915328 bytes 2026-02-22T21:56:00.762 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15900 IOs, 6319558656 bytes 2026-02-22T21:56:00.818 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16000 IOs, 6368920576 bytes 2026-02-22T21:56:00.860 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16100 IOs, 6398654464 bytes 2026-02-22T21:56:00.897 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16200 IOs, 6413466624 bytes 2026-02-22T21:56:00.947 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16300 IOs, 6459714560 bytes 2026-02-22T21:56:00.995 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16400 IOs, 6496642048 bytes 2026-02-22T21:56:01.068 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16500 IOs, 6582569984 bytes 2026-02-22T21:56:01.096 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16600 IOs, 6591356928 bytes 2026-02-22T21:56:01.133 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16700 IOs, 6601248768 bytes 2026-02-22T21:56:01.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16800 IOs, 6633836544 bytes 2026-02-22T21:56:01.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16900 IOs, 6639420416 bytes 2026-02-22T21:56:01.258 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17000 IOs, 6697942016 bytes 2026-02-22T21:56:01.296 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17100 IOs, 6719370240 bytes 2026-02-22T21:56:01.338 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17200 IOs, 6759091200 bytes 2026-02-22T21:56:01.395 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17300 IOs, 6799384576 bytes 2026-02-22T21:56:01.425 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17400 IOs, 6811904000 bytes 2026-02-22T21:56:01.492 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17500 IOs, 6881345536 bytes 2026-02-22T21:56:01.535 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17600 IOs, 6914806784 bytes 2026-02-22T21:56:01.591 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17700 IOs, 6988878848 bytes 2026-02-22T21:56:01.645 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17800 IOs, 7024710656 bytes 2026-02-22T21:56:01.694 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17900 IOs, 7065132032 bytes 2026-02-22T21:56:01.746 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18000 IOs, 7117364224 bytes 2026-02-22T21:56:01.803 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18100 IOs, 7184577536 bytes 2026-02-22T21:56:01.863 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18200 IOs, 7246391296 bytes 2026-02-22T21:56:01.924 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18300 IOs, 7320550400 bytes 2026-02-22T21:56:01.967 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18400 IOs, 7355665408 bytes 2026-02-22T21:56:02.021 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18500 IOs, 7398301696 bytes 2026-02-22T21:56:02.057 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18600 IOs, 7421789184 bytes 2026-02-22T21:56:02.108 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18700 IOs, 7465295872 bytes 2026-02-22T21:56:02.154 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18800 IOs, 7497954304 bytes 2026-02-22T21:56:02.225 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18900 IOs, 7578417152 bytes 2026-02-22T21:56:02.276 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19000 IOs, 7634315264 bytes 2026-02-22T21:56:02.316 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19100 IOs, 7656973312 bytes 2026-02-22T21:56:02.355 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19200 IOs, 7674059776 bytes 2026-02-22T21:56:02.427 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19300 IOs, 7758938112 bytes 2026-02-22T21:56:02.460 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19400 IOs, 7760649216 bytes 2026-02-22T21:56:02.490 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19500 IOs, 7770978304 bytes 2026-02-22T21:56:02.529 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19600 IOs, 7793333248 bytes 2026-02-22T21:56:02.568 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19700 IOs, 7830995968 bytes 2026-02-22T21:56:02.666 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19800 IOs, 7992261632 bytes 2026-02-22T21:56:02.715 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19900 IOs, 8022419456 bytes 2026-02-22T21:56:02.753 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20000 IOs, 8041725952 bytes 2026-02-22T21:56:02.807 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20100 IOs, 8114922496 bytes 2026-02-22T21:56:02.863 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20200 IOs, 8151568384 bytes 2026-02-22T21:56:02.916 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20300 IOs, 8197929984 bytes 2026-02-22T21:56:02.985 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20400 IOs, 8287203328 bytes 2026-02-22T21:56:03.030 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20500 IOs, 8304562176 bytes 2026-02-22T21:56:03.095 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20600 IOs, 8381288448 bytes 2026-02-22T21:56:03.133 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20700 IOs, 8404247552 bytes 2026-02-22T21:56:03.160 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20800 IOs, 8407279616 bytes 2026-02-22T21:56:03.213 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20900 IOs, 8464924672 bytes 2026-02-22T21:56:03.256 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21000 IOs, 8492605440 bytes 2026-02-22T21:56:03.324 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21100 IOs, 8586256384 bytes 2026-02-22T21:56:03.372 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21200 IOs, 8622105600 bytes 2026-02-22T21:56:03.433 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21300 IOs, 8677949440 bytes 2026-02-22T21:56:03.466 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21400 IOs, 8687235072 bytes 2026-02-22T21:56:03.509 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21500 IOs, 8713752576 bytes 2026-02-22T21:56:03.541 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21600 IOs, 8725339136 bytes 2026-02-22T21:56:03.576 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21700 IOs, 8747707392 bytes 2026-02-22T21:56:03.613 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21800 IOs, 8769256448 bytes 2026-02-22T21:56:03.676 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21900 IOs, 8846185472 bytes 2026-02-22T21:56:03.711 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22000 IOs, 8869787648 bytes 2026-02-22T21:56:03.766 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22100 IOs, 8912188416 bytes 2026-02-22T21:56:03.800 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22200 IOs, 8928926720 bytes 2026-02-22T21:56:03.849 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22300 IOs, 8972136448 bytes 2026-02-22T21:56:03.902 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22400 IOs, 9034077184 bytes 2026-02-22T21:56:03.942 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22500 IOs, 9064502272 bytes 2026-02-22T21:56:03.995 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22600 IOs, 9127520256 bytes 2026-02-22T21:56:04.077 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22700 IOs, 9214530560 bytes 2026-02-22T21:56:04.123 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22800 IOs, 9253638144 bytes 2026-02-22T21:56:04.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22900 IOs, 9326733312 bytes 2026-02-22T21:56:04.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23000 IOs, 9397971968 bytes 2026-02-22T21:56:04.281 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23100 IOs, 9413254144 bytes 2026-02-22T21:56:04.343 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23200 IOs, 9491304448 bytes 2026-02-22T21:56:04.398 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23300 IOs, 9536742400 bytes 2026-02-22T21:56:04.459 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23400 IOs, 9594560512 bytes 2026-02-22T21:56:04.510 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23500 IOs, 9656956928 bytes 2026-02-22T21:56:04.568 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23600 IOs, 9722428416 bytes 2026-02-22T21:56:04.612 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23700 IOs, 9752983552 bytes 2026-02-22T21:56:04.651 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23800 IOs, 9779025920 bytes 2026-02-22T21:56:04.713 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23900 IOs, 9862805504 bytes 2026-02-22T21:56:04.771 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24000 IOs, 9913610240 bytes 2026-02-22T21:56:04.806 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24100 IOs, 9924927488 bytes 2026-02-22T21:56:04.845 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24200 IOs, 9954186240 bytes 2026-02-22T21:56:04.876 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24300 IOs, 9963190272 bytes 2026-02-22T21:56:04.915 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24400 IOs, 9985597440 bytes 2026-02-22T21:56:04.944 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24500 IOs, 9992039424 bytes 2026-02-22T21:56:04.984 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24600 IOs, 10016247808 bytes 2026-02-22T21:56:04.985 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.1 2026-02-22T21:56:04.985 INFO:teuthology.orchestra.run:waiting for 600 2026-02-22T21:56:04.985 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.1 2026-02-22T21:56:04.986 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Restarting daemon 2026-02-22T21:56:04.986 DEBUG:teuthology.orchestra.run.trial026:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 2026-02-22T21:56:04.988 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Started 2026-02-22T21:56:04.989 INFO:tasks.rbd_mirror_thrash:waiting for 70.31163803355444 secs before thrashing 2026-02-22T21:56:05.033 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24700 IOs, 10046332928 bytes 2026-02-22T21:56:05.094 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24800 IOs, 10098341888 bytes 2026-02-22T21:56:05.127 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24900 IOs, 10098956288 bytes 2026-02-22T21:56:05.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25000 IOs, 10124314624 bytes 2026-02-22T21:56:05.222 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25100 IOs, 10159877120 bytes 2026-02-22T21:56:05.267 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25200 IOs, 10192715776 bytes 2026-02-22T21:56:05.312 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25300 IOs, 10230048768 bytes 2026-02-22T21:56:05.348 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25400 IOs, 10248148992 bytes 2026-02-22T21:56:05.382 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25500 IOs, 10265997312 bytes 2026-02-22T21:56:05.445 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25600 IOs, 10326522880 bytes 2026-02-22T21:56:05.498 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25700 IOs, 10394569728 bytes 2026-02-22T21:56:05.548 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25800 IOs, 10425466880 bytes 2026-02-22T21:56:05.585 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25900 IOs, 10469707776 bytes 2026-02-22T21:56:05.630 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:26000 IOs, 10495082496 bytes 2026-02-22T21:56:05.669 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:26100 IOs, 10527421440 bytes 2026-02-22T21:56:05.723 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:26200 IOs, 10572142592 bytes 2026-02-22T21:56:05.761 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:26300 IOs, 10587501568 bytes 2026-02-22T21:56:05.792 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:26400 IOs, 10593614848 bytes 2026-02-22T21:56:05.840 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:26500 IOs, 10630618112 bytes 2026-02-22T21:56:05.868 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:26600 IOs, 10633690112 bytes 2026-02-22T21:56:05.905 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:26700 IOs, 10655855616 bytes 2026-02-22T21:56:05.956 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:26800 IOs, 10701712384 bytes 2026-02-22T21:56:06.000 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:26900 IOs, 10734013440 bytes 2026-02-22T21:56:06.073 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:27000 IOs, 10822881280 bytes 2026-02-22T21:56:06.104 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:27100 IOs, 10824342528 bytes 2026-02-22T21:56:06.154 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:27200 IOs, 10872376320 bytes 2026-02-22T21:56:06.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:27300 IOs, 10900913152 bytes 2026-02-22T21:56:06.245 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:27400 IOs, 10944140288 bytes 2026-02-22T21:56:06.317 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:27500 IOs, 11012184064 bytes 2026-02-22T21:56:06.392 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:27600 IOs, 11105994752 bytes 2026-02-22T21:56:06.441 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:27700 IOs, 11162042368 bytes 2026-02-22T21:56:06.503 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:27800 IOs, 11194295296 bytes 2026-02-22T21:56:06.547 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:27900 IOs, 11219387392 bytes 2026-02-22T21:56:06.581 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:28000 IOs, 11233626112 bytes 2026-02-22T21:56:06.621 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:28100 IOs, 11259493376 bytes 2026-02-22T21:56:06.662 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:28200 IOs, 11276820480 bytes 2026-02-22T21:56:06.690 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:28300 IOs, 11277415424 bytes 2026-02-22T21:56:06.744 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:28400 IOs, 11327232000 bytes 2026-02-22T21:56:06.779 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:28500 IOs, 11338943488 bytes 2026-02-22T21:56:06.823 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:28600 IOs, 11376374784 bytes 2026-02-22T21:56:06.879 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:28700 IOs, 11426585600 bytes 2026-02-22T21:56:06.926 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:28800 IOs, 11455981568 bytes 2026-02-22T21:56:06.980 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:28900 IOs, 11513044992 bytes 2026-02-22T21:56:07.012 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:29000 IOs, 11524750336 bytes 2026-02-22T21:56:07.070 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:29100 IOs, 11579789312 bytes 2026-02-22T21:56:07.157 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:29200 IOs, 11708088320 bytes 2026-02-22T21:56:07.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ error_code=124 2026-02-22T21:56:07.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set -e 2026-02-22T21:56:07.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' 124 -eq 124 ']' 2026-02-22T21:56:07.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:56:07.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T21:56:07.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_49 2026-02-22T21:56:07.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image_and_enable_mirror cluster2 mirror image_49 snapshot 128M 2026-02-22T21:56:07.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:56:07.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:56:07.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:56:07.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:56:07.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_49 2026-02-22T21:56:07.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:56:07.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T21:56:07.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n snapshot ']' 2026-02-22T21:56:07.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:56:07.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image cluster2 mirror image_49 128M 2026-02-22T21:56:07.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:56:07.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:56:07.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:56:07.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:56:07.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_49 2026-02-22T21:56:07.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:56:07.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local size=128 2026-02-22T21:56:07.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n 128M ']' 2026-02-22T21:56:07.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ size=128M 2026-02-22T21:56:07.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:56:07.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_49 2026-02-22T21:56:07.241 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T21:56:07.241 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:56:07.241 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' image = image ']' 2026-02-22T21:56:07.241 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:56:07.241 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ enable_mirror cluster2 mirror image_49 snapshot 2026-02-22T21:56:07.241 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:56:07.241 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:56:07.241 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_49 2026-02-22T21:56:07.241 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T21:56:07.241 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_49 snapshot 2026-02-22T21:56:08.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Mirroring enabled 2026-02-22T21:56:08.187 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 info mirror/image_49 2026-02-22T21:56:08.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_49': 2026-02-22T21:56:08.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 128 MiB in 32 objects 2026-02-22T21:56:08.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T21:56:08.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 1 2026-02-22T21:56:08.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 149f514cc355 2026-02-22T21:56:08.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.149f514cc355 2026-02-22T21:56:08.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T21:56:08.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock 2026-02-22T21:56:08.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T21:56:08.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T21:56:08.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:56:07 2026 2026-02-22T21:56:08.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:56:07 2026 2026-02-22T21:56:08.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:56:07 2026 2026-02-22T21:56:08.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T21:56:08.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T21:56:08.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: 61a34d68-de2f-48ac-99bc-627b57103d64 2026-02-22T21:56:08.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: true 2026-02-22T21:56:08.213 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n '' ']' 2026-02-22T21:56:08.213 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ stress_write_image cluster2 mirror image_49 2026-02-22T21:56:08.213 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:56:08.213 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:56:08.213 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_49 2026-02-22T21:56:08.213 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-22T21:56:08.214 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local duration=8 2026-02-22T21:56:08.214 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set +e 2026-02-22T21:56:08.214 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ timeout 8s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_49 --debug-rbd=20 --debug-journaler=20 2026-02-22T21:56:08.292 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:100 IOs, 28443648 bytes 2026-02-22T21:56:08.333 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:200 IOs, 57327616 bytes 2026-02-22T21:56:08.386 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:300 IOs, 117195776 bytes 2026-02-22T21:56:08.439 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:400 IOs, 151805952 bytes 2026-02-22T21:56:08.464 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:500 IOs, 152422400 bytes 2026-02-22T21:56:08.532 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:600 IOs, 251307008 bytes 2026-02-22T21:56:08.566 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:700 IOs, 252009472 bytes 2026-02-22T21:56:08.619 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:800 IOs, 313847808 bytes 2026-02-22T21:56:08.644 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:900 IOs, 315993088 bytes 2026-02-22T21:56:08.693 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1000 IOs, 370485248 bytes 2026-02-22T21:56:08.750 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1100 IOs, 427092992 bytes 2026-02-22T21:56:08.798 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1200 IOs, 460137472 bytes 2026-02-22T21:56:08.853 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1300 IOs, 520075264 bytes 2026-02-22T21:56:08.920 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1400 IOs, 600668160 bytes 2026-02-22T21:56:08.945 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1500 IOs, 601269248 bytes 2026-02-22T21:56:08.982 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1600 IOs, 631923712 bytes 2026-02-22T21:56:09.047 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1700 IOs, 712294400 bytes 2026-02-22T21:56:09.091 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1800 IOs, 729631744 bytes 2026-02-22T21:56:09.340 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1900 IOs, 771094528 bytes 2026-02-22T21:56:09.383 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2000 IOs, 814058496 bytes 2026-02-22T21:56:09.417 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2100 IOs, 814668800 bytes 2026-02-22T21:56:09.462 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2200 IOs, 828796928 bytes 2026-02-22T21:56:09.524 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2300 IOs, 902661120 bytes 2026-02-22T21:56:09.570 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2400 IOs, 936310784 bytes 2026-02-22T21:56:09.620 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2500 IOs, 968498176 bytes 2026-02-22T21:56:09.664 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2600 IOs, 1002380288 bytes 2026-02-22T21:56:09.731 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2700 IOs, 1081040896 bytes 2026-02-22T21:56:09.774 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2800 IOs, 1118552064 bytes 2026-02-22T21:56:09.812 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2900 IOs, 1140876288 bytes 2026-02-22T21:56:09.857 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3000 IOs, 1175361536 bytes 2026-02-22T21:56:09.910 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3100 IOs, 1230370816 bytes 2026-02-22T21:56:09.961 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3200 IOs, 1293696000 bytes 2026-02-22T21:56:10.014 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3300 IOs, 1330591744 bytes 2026-02-22T21:56:10.054 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3400 IOs, 1357265920 bytes 2026-02-22T21:56:10.118 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3500 IOs, 1422394368 bytes 2026-02-22T21:56:10.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3600 IOs, 1459788800 bytes 2026-02-22T21:56:10.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3700 IOs, 1480315904 bytes 2026-02-22T21:56:10.229 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3800 IOs, 1494451200 bytes 2026-02-22T21:56:10.317 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3900 IOs, 1635127296 bytes 2026-02-22T21:56:10.365 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4000 IOs, 1663168512 bytes 2026-02-22T21:56:10.394 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4100 IOs, 1666343936 bytes 2026-02-22T21:56:10.459 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4200 IOs, 1733266432 bytes 2026-02-22T21:56:10.503 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4300 IOs, 1758045184 bytes 2026-02-22T21:56:10.539 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4400 IOs, 1774541824 bytes 2026-02-22T21:56:10.611 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4500 IOs, 1857925120 bytes 2026-02-22T21:56:10.648 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4600 IOs, 1881379840 bytes 2026-02-22T21:56:10.693 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4700 IOs, 1922889728 bytes 2026-02-22T21:56:10.758 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4800 IOs, 1998559232 bytes 2026-02-22T21:56:10.806 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4900 IOs, 2052643840 bytes 2026-02-22T21:56:10.882 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5000 IOs, 2145147904 bytes 2026-02-22T21:56:10.929 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5100 IOs, 2193499136 bytes 2026-02-22T21:56:11.002 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5200 IOs, 2285394944 bytes 2026-02-22T21:56:11.037 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5300 IOs, 2299425792 bytes 2026-02-22T21:56:11.074 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5400 IOs, 2322644992 bytes 2026-02-22T21:56:11.131 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5500 IOs, 2376804352 bytes 2026-02-22T21:56:11.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5600 IOs, 2469962752 bytes 2026-02-22T21:56:11.246 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5700 IOs, 2487207936 bytes 2026-02-22T21:56:11.280 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5800 IOs, 2499812352 bytes 2026-02-22T21:56:11.310 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5900 IOs, 2506806272 bytes 2026-02-22T21:56:11.337 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6000 IOs, 2523809792 bytes 2026-02-22T21:56:11.408 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6100 IOs, 2591248384 bytes 2026-02-22T21:56:11.465 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6200 IOs, 2633598976 bytes 2026-02-22T21:56:11.516 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6300 IOs, 2678276096 bytes 2026-02-22T21:56:11.580 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6400 IOs, 2749975552 bytes 2026-02-22T21:56:11.647 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6500 IOs, 2833579008 bytes 2026-02-22T21:56:11.693 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6600 IOs, 2868390912 bytes 2026-02-22T21:56:11.736 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6700 IOs, 2898397184 bytes 2026-02-22T21:56:11.789 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6800 IOs, 2945342464 bytes 2026-02-22T21:56:11.827 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6900 IOs, 2966658048 bytes 2026-02-22T21:56:11.866 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7000 IOs, 3000774656 bytes 2026-02-22T21:56:11.932 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7100 IOs, 3064591360 bytes 2026-02-22T21:56:12.000 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7200 IOs, 3144827904 bytes 2026-02-22T21:56:12.066 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7300 IOs, 3239521280 bytes 2026-02-22T21:56:12.129 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7400 IOs, 3288108032 bytes 2026-02-22T21:56:12.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7500 IOs, 3345028096 bytes 2026-02-22T21:56:12.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7600 IOs, 3396176896 bytes 2026-02-22T21:56:12.294 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7700 IOs, 3454906368 bytes 2026-02-22T21:56:12.330 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7800 IOs, 3467478016 bytes 2026-02-22T21:56:12.369 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7900 IOs, 3491985408 bytes 2026-02-22T21:56:12.407 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8000 IOs, 3515407360 bytes 2026-02-22T21:56:12.452 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8100 IOs, 3551680512 bytes 2026-02-22T21:56:12.499 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8200 IOs, 3591781376 bytes 2026-02-22T21:56:12.536 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8300 IOs, 3613089792 bytes 2026-02-22T21:56:12.580 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8400 IOs, 3645927424 bytes 2026-02-22T21:56:12.611 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8500 IOs, 3651809280 bytes 2026-02-22T21:56:12.681 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8600 IOs, 3733765120 bytes 2026-02-22T21:56:12.737 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8700 IOs, 3796109312 bytes 2026-02-22T21:56:12.807 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8800 IOs, 3894196224 bytes 2026-02-22T21:56:12.863 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8900 IOs, 3943288832 bytes 2026-02-22T21:56:12.899 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9000 IOs, 3982112768 bytes 2026-02-22T21:56:12.956 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9100 IOs, 4043354112 bytes 2026-02-22T21:56:13.008 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9200 IOs, 4073327616 bytes 2026-02-22T21:56:13.047 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9300 IOs, 4095226880 bytes 2026-02-22T21:56:13.089 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9400 IOs, 4128466944 bytes 2026-02-22T21:56:13.149 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9500 IOs, 4194634752 bytes 2026-02-22T21:56:13.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9600 IOs, 4248482816 bytes 2026-02-22T21:56:13.243 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9700 IOs, 4268465152 bytes 2026-02-22T21:56:13.275 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9800 IOs, 4289229824 bytes 2026-02-22T21:56:13.338 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9900 IOs, 4363286528 bytes 2026-02-22T21:56:13.368 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10000 IOs, 4371932160 bytes 2026-02-22T21:56:13.397 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10100 IOs, 4378205184 bytes 2026-02-22T21:56:13.433 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10200 IOs, 4401060864 bytes 2026-02-22T21:56:13.498 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10300 IOs, 4464969728 bytes 2026-02-22T21:56:13.545 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10400 IOs, 4499095552 bytes 2026-02-22T21:56:13.593 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10500 IOs, 4531671040 bytes 2026-02-22T21:56:13.636 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10600 IOs, 4572738560 bytes 2026-02-22T21:56:13.700 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10700 IOs, 4651859968 bytes 2026-02-22T21:56:13.765 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10800 IOs, 4728707072 bytes 2026-02-22T21:56:13.802 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10900 IOs, 4764677120 bytes 2026-02-22T21:56:13.855 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11000 IOs, 4805809152 bytes 2026-02-22T21:56:13.894 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11100 IOs, 4831926272 bytes 2026-02-22T21:56:13.930 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11200 IOs, 4848189440 bytes 2026-02-22T21:56:13.981 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11300 IOs, 4896530432 bytes 2026-02-22T21:56:14.032 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11400 IOs, 4939581440 bytes 2026-02-22T21:56:14.075 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11500 IOs, 4966569984 bytes 2026-02-22T21:56:14.146 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11600 IOs, 5038921728 bytes 2026-02-22T21:56:14.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11700 IOs, 5059131392 bytes 2026-02-22T21:56:14.231 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11800 IOs, 5090078720 bytes 2026-02-22T21:56:14.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11900 IOs, 5156205568 bytes 2026-02-22T21:56:14.317 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12000 IOs, 5156820992 bytes 2026-02-22T21:56:14.361 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12100 IOs, 5193999360 bytes 2026-02-22T21:56:14.405 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12200 IOs, 5224261632 bytes 2026-02-22T21:56:14.440 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12300 IOs, 5241715712 bytes 2026-02-22T21:56:14.495 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12400 IOs, 5292710912 bytes 2026-02-22T21:56:14.572 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12500 IOs, 5374999552 bytes 2026-02-22T21:56:14.611 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12600 IOs, 5399984128 bytes 2026-02-22T21:56:14.655 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12700 IOs, 5437628416 bytes 2026-02-22T21:56:14.687 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12800 IOs, 5448821760 bytes 2026-02-22T21:56:14.739 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12900 IOs, 5504130048 bytes 2026-02-22T21:56:14.802 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13000 IOs, 5588219904 bytes 2026-02-22T21:56:14.853 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13100 IOs, 5622586368 bytes 2026-02-22T21:56:14.898 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13200 IOs, 5667059712 bytes 2026-02-22T21:56:14.944 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13300 IOs, 5688305664 bytes 2026-02-22T21:56:15.003 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13400 IOs, 5755470848 bytes 2026-02-22T21:56:15.062 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13500 IOs, 5818677248 bytes 2026-02-22T21:56:15.129 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13600 IOs, 5907887104 bytes 2026-02-22T21:56:15.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13700 IOs, 5948395520 bytes 2026-02-22T21:56:15.249 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13800 IOs, 6048098304 bytes 2026-02-22T21:56:15.293 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13900 IOs, 6072928256 bytes 2026-02-22T21:56:15.320 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14000 IOs, 6077846528 bytes 2026-02-22T21:56:15.370 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14100 IOs, 6138140672 bytes 2026-02-22T21:56:15.428 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14200 IOs, 6207199232 bytes 2026-02-22T21:56:15.470 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14300 IOs, 6230024192 bytes 2026-02-22T21:56:15.524 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14400 IOs, 6256596992 bytes 2026-02-22T21:56:15.551 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14500 IOs, 6265529344 bytes 2026-02-22T21:56:15.590 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14600 IOs, 6285095936 bytes 2026-02-22T21:56:15.649 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14700 IOs, 6353304576 bytes 2026-02-22T21:56:15.688 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14800 IOs, 6381832192 bytes 2026-02-22T21:56:15.737 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14900 IOs, 6421420032 bytes 2026-02-22T21:56:15.776 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15000 IOs, 6458087424 bytes 2026-02-22T21:56:15.860 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15100 IOs, 6555622400 bytes 2026-02-22T21:56:15.901 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15200 IOs, 6584601600 bytes 2026-02-22T21:56:15.934 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15300 IOs, 6618009600 bytes 2026-02-22T21:56:16.012 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15400 IOs, 6699508736 bytes 2026-02-22T21:56:16.070 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15500 IOs, 6776022016 bytes 2026-02-22T21:56:16.127 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15600 IOs, 6824834048 bytes 2026-02-22T21:56:16.163 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15700 IOs, 6831491072 bytes 2026-02-22T21:56:16.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ error_code=124 2026-02-22T21:56:16.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set -e 2026-02-22T21:56:16.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' 124 -eq 124 ']' 2026-02-22T21:56:16.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:56:16.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T21:56:16.222 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_50 2026-02-22T21:56:16.222 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image_and_enable_mirror cluster2 mirror image_50 snapshot 128M 2026-02-22T21:56:16.222 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:56:16.222 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:56:16.222 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:56:16.222 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:56:16.222 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_50 2026-02-22T21:56:16.222 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:56:16.222 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T21:56:16.222 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n snapshot ']' 2026-02-22T21:56:16.222 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:56:16.222 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image cluster2 mirror image_50 128M 2026-02-22T21:56:16.223 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:56:16.223 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:56:16.223 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:56:16.223 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:56:16.223 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_50 2026-02-22T21:56:16.223 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:56:16.223 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local size=128 2026-02-22T21:56:16.223 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n 128M ']' 2026-02-22T21:56:16.223 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ size=128M 2026-02-22T21:56:16.223 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T21:56:16.223 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_50 2026-02-22T21:56:16.252 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T21:56:16.252 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:56:16.252 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' image = image ']' 2026-02-22T21:56:16.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:56:16.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ enable_mirror cluster2 mirror image_50 snapshot 2026-02-22T21:56:16.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:56:16.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:56:16.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_50 2026-02-22T21:56:16.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T21:56:16.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_50 snapshot 2026-02-22T21:56:17.188 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Mirroring enabled 2026-02-22T21:56:17.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 info mirror/image_50 2026-02-22T21:56:17.218 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_50': 2026-02-22T21:56:17.218 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 128 MiB in 32 objects 2026-02-22T21:56:17.218 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T21:56:17.218 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 1 2026-02-22T21:56:17.218 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 14a9726417d1 2026-02-22T21:56:17.218 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.14a9726417d1 2026-02-22T21:56:17.218 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T21:56:17.218 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock 2026-02-22T21:56:17.218 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T21:56:17.218 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T21:56:17.218 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:56:16 2026 2026-02-22T21:56:17.218 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:56:16 2026 2026-02-22T21:56:17.219 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:56:16 2026 2026-02-22T21:56:17.219 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T21:56:17.219 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T21:56:17.219 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: 319a0448-b0a9-483d-b56f-b265254e2f2e 2026-02-22T21:56:17.219 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: true 2026-02-22T21:56:17.220 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n '' ']' 2026-02-22T21:56:17.220 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ stress_write_image cluster2 mirror image_50 2026-02-22T21:56:17.220 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:56:17.220 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:56:17.220 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_50 2026-02-22T21:56:17.220 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-22T21:56:17.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local duration=14 2026-02-22T21:56:17.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set +e 2026-02-22T21:56:17.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ timeout 14s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_50 --debug-rbd=20 --debug-journaler=20 2026-02-22T21:56:17.291 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:100 IOs, 22556672 bytes 2026-02-22T21:56:17.324 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:200 IOs, 37448704 bytes 2026-02-22T21:56:17.364 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:300 IOs, 61553664 bytes 2026-02-22T21:56:17.407 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:400 IOs, 104521728 bytes 2026-02-22T21:56:17.445 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:500 IOs, 113142784 bytes 2026-02-22T21:56:17.477 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:600 IOs, 127050752 bytes 2026-02-22T21:56:17.506 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:700 IOs, 127654912 bytes 2026-02-22T21:56:17.549 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:800 IOs, 151476224 bytes 2026-02-22T21:56:17.600 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:900 IOs, 198461440 bytes 2026-02-22T21:56:17.651 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1000 IOs, 251939840 bytes 2026-02-22T21:56:17.692 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1100 IOs, 276447232 bytes 2026-02-22T21:56:17.724 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1200 IOs, 286761984 bytes 2026-02-22T21:56:17.782 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1300 IOs, 342554624 bytes 2026-02-22T21:56:17.819 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1400 IOs, 362747904 bytes 2026-02-22T21:56:17.865 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1500 IOs, 410463232 bytes 2026-02-22T21:56:17.917 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1600 IOs, 458320896 bytes 2026-02-22T21:56:17.980 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1700 IOs, 534933504 bytes 2026-02-22T21:56:18.041 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1800 IOs, 587867136 bytes 2026-02-22T21:56:18.077 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1900 IOs, 604643328 bytes 2026-02-22T21:56:18.119 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2000 IOs, 637804544 bytes 2026-02-22T21:56:18.150 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2100 IOs, 642810880 bytes 2026-02-22T21:56:18.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2200 IOs, 718606336 bytes 2026-02-22T21:56:18.267 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2300 IOs, 778717184 bytes 2026-02-22T21:56:18.312 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2400 IOs, 826627072 bytes 2026-02-22T21:56:18.350 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2500 IOs, 850159616 bytes 2026-02-22T21:56:18.412 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2600 IOs, 938116096 bytes 2026-02-22T21:56:18.465 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2700 IOs, 982629376 bytes 2026-02-22T21:56:18.558 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2800 IOs, 1105839104 bytes 2026-02-22T21:56:18.601 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2900 IOs, 1132303360 bytes 2026-02-22T21:56:18.634 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3000 IOs, 1146059776 bytes 2026-02-22T21:56:18.674 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3100 IOs, 1170121728 bytes 2026-02-22T21:56:18.721 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3200 IOs, 1199306752 bytes 2026-02-22T21:56:18.754 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3300 IOs, 1210536960 bytes 2026-02-22T21:56:18.802 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3400 IOs, 1270107136 bytes 2026-02-22T21:56:18.851 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3500 IOs, 1311769600 bytes 2026-02-22T21:56:18.901 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3600 IOs, 1351762944 bytes 2026-02-22T21:56:18.954 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3700 IOs, 1399748608 bytes 2026-02-22T21:56:18.996 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3800 IOs, 1435346944 bytes 2026-02-22T21:56:19.040 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3900 IOs, 1465864192 bytes 2026-02-22T21:56:19.085 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4000 IOs, 1489610752 bytes 2026-02-22T21:56:19.132 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4100 IOs, 1524389888 bytes 2026-02-22T21:56:19.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4200 IOs, 1553299456 bytes 2026-02-22T21:56:19.234 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4300 IOs, 1602791424 bytes 2026-02-22T21:56:19.300 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4400 IOs, 1683989504 bytes 2026-02-22T21:56:19.336 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4500 IOs, 1713538048 bytes 2026-02-22T21:56:19.409 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4600 IOs, 1837309952 bytes 2026-02-22T21:56:19.489 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4700 IOs, 1934111744 bytes 2026-02-22T21:56:19.549 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4800 IOs, 1982840832 bytes 2026-02-22T21:56:19.605 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4900 IOs, 2038280192 bytes 2026-02-22T21:56:19.646 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5000 IOs, 2062222336 bytes 2026-02-22T21:56:19.704 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5100 IOs, 2125292544 bytes 2026-02-22T21:56:19.745 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5200 IOs, 2157599744 bytes 2026-02-22T21:56:19.782 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5300 IOs, 2176337920 bytes 2026-02-22T21:56:19.826 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5400 IOs, 2238346240 bytes 2026-02-22T21:56:19.904 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5500 IOs, 2318182400 bytes 2026-02-22T21:56:19.946 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5600 IOs, 2377522176 bytes 2026-02-22T21:56:20.005 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5700 IOs, 2419605504 bytes 2026-02-22T21:56:20.067 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5800 IOs, 2497653760 bytes 2026-02-22T21:56:20.101 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5900 IOs, 2508485632 bytes 2026-02-22T21:56:20.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6000 IOs, 2626612224 bytes 2026-02-22T21:56:20.245 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6100 IOs, 2710840320 bytes 2026-02-22T21:56:20.305 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6200 IOs, 2771162112 bytes 2026-02-22T21:56:20.361 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6300 IOs, 2826252288 bytes 2026-02-22T21:56:20.404 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6400 IOs, 2858189824 bytes 2026-02-22T21:56:20.461 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6500 IOs, 2923830272 bytes 2026-02-22T21:56:20.526 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6600 IOs, 2984122368 bytes 2026-02-22T21:56:20.572 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6700 IOs, 3030322176 bytes 2026-02-22T21:56:20.627 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6800 IOs, 3073796096 bytes 2026-02-22T21:56:20.673 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6900 IOs, 3121805312 bytes 2026-02-22T21:56:20.731 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7000 IOs, 3184602112 bytes 2026-02-22T21:56:20.772 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7100 IOs, 3208017920 bytes 2026-02-22T21:56:20.807 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7200 IOs, 3238016000 bytes 2026-02-22T21:56:20.849 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7300 IOs, 3261900800 bytes 2026-02-22T21:56:20.911 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7400 IOs, 3344625664 bytes 2026-02-22T21:56:20.950 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7500 IOs, 3384968192 bytes 2026-02-22T21:56:21.012 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7600 IOs, 3431930880 bytes 2026-02-22T21:56:21.058 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7700 IOs, 3470540800 bytes 2026-02-22T21:56:21.095 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7800 IOs, 3483762688 bytes 2026-02-22T21:56:21.146 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7900 IOs, 3522465792 bytes 2026-02-22T21:56:21.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8000 IOs, 3560231936 bytes 2026-02-22T21:56:21.250 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8100 IOs, 3613902848 bytes 2026-02-22T21:56:21.324 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8200 IOs, 3744959488 bytes 2026-02-22T21:56:21.391 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8300 IOs, 3801784320 bytes 2026-02-22T21:56:21.430 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8400 IOs, 3822639104 bytes 2026-02-22T21:56:21.500 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8500 IOs, 3912396800 bytes 2026-02-22T21:56:21.550 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8600 IOs, 3946957824 bytes 2026-02-22T21:56:21.590 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8700 IOs, 3968031744 bytes 2026-02-22T21:56:21.648 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8800 IOs, 4036517888 bytes 2026-02-22T21:56:21.701 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8900 IOs, 4087446528 bytes 2026-02-22T21:56:21.762 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9000 IOs, 4184680448 bytes 2026-02-22T21:56:21.821 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9100 IOs, 4236555264 bytes 2026-02-22T21:56:21.871 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9200 IOs, 4288400384 bytes 2026-02-22T21:56:21.918 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9300 IOs, 4320503808 bytes 2026-02-22T21:56:21.952 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9400 IOs, 4345799680 bytes 2026-02-22T21:56:22.203 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9500 IOs, 4404244480 bytes 2026-02-22T21:56:22.241 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9600 IOs, 4422241280 bytes 2026-02-22T21:56:22.293 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9700 IOs, 4477649920 bytes 2026-02-22T21:56:22.345 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9800 IOs, 4526884864 bytes 2026-02-22T21:56:22.382 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9900 IOs, 4549820416 bytes 2026-02-22T21:56:22.437 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10000 IOs, 4617911296 bytes 2026-02-22T21:56:22.496 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10100 IOs, 4682151936 bytes 2026-02-22T21:56:22.545 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10200 IOs, 4696933376 bytes 2026-02-22T21:56:22.601 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10300 IOs, 4756142080 bytes 2026-02-22T21:56:22.642 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10400 IOs, 4772722688 bytes 2026-02-22T21:56:22.675 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10500 IOs, 4804631552 bytes 2026-02-22T21:56:22.732 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10600 IOs, 4854902784 bytes 2026-02-22T21:56:22.798 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10700 IOs, 4907414528 bytes 2026-02-22T21:56:22.843 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10800 IOs, 4943250432 bytes 2026-02-22T21:56:22.884 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10900 IOs, 4968410112 bytes 2026-02-22T21:56:22.934 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11000 IOs, 5014756352 bytes 2026-02-22T21:56:23.007 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11100 IOs, 5120421888 bytes 2026-02-22T21:56:23.053 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11200 IOs, 5149241344 bytes 2026-02-22T21:56:23.104 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11300 IOs, 5188133888 bytes 2026-02-22T21:56:23.141 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11400 IOs, 5202796544 bytes 2026-02-22T21:56:23.171 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11500 IOs, 5209822208 bytes 2026-02-22T21:56:23.223 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11600 IOs, 5258483712 bytes 2026-02-22T21:56:23.268 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11700 IOs, 5305574400 bytes 2026-02-22T21:56:23.330 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11800 IOs, 5353021440 bytes 2026-02-22T21:56:23.390 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11900 IOs, 5416909824 bytes 2026-02-22T21:56:23.434 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12000 IOs, 5468359680 bytes 2026-02-22T21:56:23.487 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12100 IOs, 5509764096 bytes 2026-02-22T21:56:23.532 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12200 IOs, 5543604224 bytes 2026-02-22T21:56:23.570 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12300 IOs, 5572110336 bytes 2026-02-22T21:56:23.614 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12400 IOs, 5593689088 bytes 2026-02-22T21:56:23.666 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12500 IOs, 5628460032 bytes 2026-02-22T21:56:23.712 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12600 IOs, 5659555840 bytes 2026-02-22T21:56:23.767 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12700 IOs, 5708348416 bytes 2026-02-22T21:56:23.821 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12800 IOs, 5748754432 bytes 2026-02-22T21:56:23.881 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12900 IOs, 5805975552 bytes 2026-02-22T21:56:23.931 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13000 IOs, 5840775168 bytes 2026-02-22T21:56:23.977 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13100 IOs, 5872968704 bytes 2026-02-22T21:56:24.010 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13200 IOs, 5884943360 bytes 2026-02-22T21:56:24.074 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13300 IOs, 5958080512 bytes 2026-02-22T21:56:24.115 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13400 IOs, 5978577920 bytes 2026-02-22T21:56:24.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13500 IOs, 6023537664 bytes 2026-02-22T21:56:24.235 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13600 IOs, 6101828608 bytes 2026-02-22T21:56:24.275 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13700 IOs, 6138137600 bytes 2026-02-22T21:56:24.320 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13800 IOs, 6157185024 bytes 2026-02-22T21:56:24.364 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13900 IOs, 6187335680 bytes 2026-02-22T21:56:24.407 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14000 IOs, 6217375744 bytes 2026-02-22T21:56:24.476 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14100 IOs, 6299450368 bytes 2026-02-22T21:56:24.559 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14200 IOs, 6402048000 bytes 2026-02-22T21:56:24.624 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14300 IOs, 6469676032 bytes 2026-02-22T21:56:24.672 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14400 IOs, 6511056896 bytes 2026-02-22T21:56:24.711 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14500 IOs, 6529846272 bytes 2026-02-22T21:56:24.766 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14600 IOs, 6584337408 bytes 2026-02-22T21:56:24.836 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14700 IOs, 6673222656 bytes 2026-02-22T21:56:24.878 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14800 IOs, 6694323200 bytes 2026-02-22T21:56:24.938 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14900 IOs, 6749622272 bytes 2026-02-22T21:56:24.993 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15000 IOs, 6831506432 bytes 2026-02-22T21:56:25.044 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15100 IOs, 6860355584 bytes 2026-02-22T21:56:25.110 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15200 IOs, 6937482240 bytes 2026-02-22T21:56:25.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15300 IOs, 6997947392 bytes 2026-02-22T21:56:25.234 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15400 IOs, 7047360512 bytes 2026-02-22T21:56:25.312 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15500 IOs, 7141307392 bytes 2026-02-22T21:56:25.370 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15600 IOs, 7221209088 bytes 2026-02-22T21:56:25.411 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15700 IOs, 7221797888 bytes 2026-02-22T21:56:25.476 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15800 IOs, 7298343936 bytes 2026-02-22T21:56:25.550 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15900 IOs, 7396600832 bytes 2026-02-22T21:56:25.601 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16000 IOs, 7452990464 bytes 2026-02-22T21:56:25.658 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16100 IOs, 7502001152 bytes 2026-02-22T21:56:25.695 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16200 IOs, 7515170816 bytes 2026-02-22T21:56:25.760 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16300 IOs, 7583210496 bytes 2026-02-22T21:56:25.799 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16400 IOs, 7608672256 bytes 2026-02-22T21:56:25.845 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16500 IOs, 7634866176 bytes 2026-02-22T21:56:25.884 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16600 IOs, 7657284608 bytes 2026-02-22T21:56:25.935 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16700 IOs, 7689011200 bytes 2026-02-22T21:56:25.988 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16800 IOs, 7740291072 bytes 2026-02-22T21:56:26.052 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16900 IOs, 7820351488 bytes 2026-02-22T21:56:26.092 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17000 IOs, 7833809920 bytes 2026-02-22T21:56:26.150 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17100 IOs, 7919052800 bytes 2026-02-22T21:56:26.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17200 IOs, 7933678592 bytes 2026-02-22T21:56:26.268 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17300 IOs, 8011393024 bytes 2026-02-22T21:56:26.318 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17400 IOs, 8032583680 bytes 2026-02-22T21:56:26.366 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17500 IOs, 8069955584 bytes 2026-02-22T21:56:26.425 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17600 IOs, 8123709440 bytes 2026-02-22T21:56:26.476 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17700 IOs, 8162772992 bytes 2026-02-22T21:56:26.510 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17800 IOs, 8168726528 bytes 2026-02-22T21:56:26.555 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17900 IOs, 8197492736 bytes 2026-02-22T21:56:26.618 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18000 IOs, 8250362880 bytes 2026-02-22T21:56:26.667 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18100 IOs, 8295579648 bytes 2026-02-22T21:56:26.741 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18200 IOs, 8386523136 bytes 2026-02-22T21:56:26.780 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18300 IOs, 8400479232 bytes 2026-02-22T21:56:26.825 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18400 IOs, 8434386944 bytes 2026-02-22T21:56:26.863 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18500 IOs, 8453796864 bytes 2026-02-22T21:56:26.923 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18600 IOs, 8516306944 bytes 2026-02-22T21:56:26.966 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18700 IOs, 8543226880 bytes 2026-02-22T21:56:27.007 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18800 IOs, 8556847104 bytes 2026-02-22T21:56:27.057 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18900 IOs, 8603287552 bytes 2026-02-22T21:56:27.093 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19000 IOs, 8603883520 bytes 2026-02-22T21:56:27.139 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19100 IOs, 8635881472 bytes 2026-02-22T21:56:27.185 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19200 IOs, 8678982656 bytes 2026-02-22T21:56:27.255 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19300 IOs, 8749942784 bytes 2026-02-22T21:56:27.298 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19400 IOs, 8776596480 bytes 2026-02-22T21:56:27.337 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19500 IOs, 8792034304 bytes 2026-02-22T21:56:27.395 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19600 IOs, 8856886272 bytes 2026-02-22T21:56:27.458 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19700 IOs, 8910158848 bytes 2026-02-22T21:56:27.518 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19800 IOs, 8992216064 bytes 2026-02-22T21:56:27.579 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19900 IOs, 9046503424 bytes 2026-02-22T21:56:27.654 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20000 IOs, 9108832256 bytes 2026-02-22T21:56:27.714 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20100 IOs, 9167475712 bytes 2026-02-22T21:56:27.744 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20200 IOs, 9171968000 bytes 2026-02-22T21:56:27.799 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20300 IOs, 9216188416 bytes 2026-02-22T21:56:27.853 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20400 IOs, 9269601280 bytes 2026-02-22T21:56:27.897 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20500 IOs, 9289473024 bytes 2026-02-22T21:56:27.957 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20600 IOs, 9349154816 bytes 2026-02-22T21:56:28.003 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20700 IOs, 9387667456 bytes 2026-02-22T21:56:28.051 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20800 IOs, 9433273344 bytes 2026-02-22T21:56:28.105 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20900 IOs, 9483596800 bytes 2026-02-22T21:56:28.144 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21000 IOs, 9502931968 bytes 2026-02-22T21:56:28.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21100 IOs, 9544600576 bytes 2026-02-22T21:56:28.263 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21200 IOs, 9620598784 bytes 2026-02-22T21:56:28.296 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21300 IOs, 9621235712 bytes 2026-02-22T21:56:28.360 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21400 IOs, 9679003648 bytes 2026-02-22T21:56:28.420 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21500 IOs, 9742752768 bytes 2026-02-22T21:56:28.468 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21600 IOs, 9783348224 bytes 2026-02-22T21:56:28.525 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21700 IOs, 9837286400 bytes 2026-02-22T21:56:28.579 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21800 IOs, 9882642432 bytes 2026-02-22T21:56:28.624 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21900 IOs, 9909206016 bytes 2026-02-22T21:56:28.680 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22000 IOs, 9968522240 bytes 2026-02-22T21:56:28.713 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22100 IOs, 9974917120 bytes 2026-02-22T21:56:28.747 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22200 IOs, 9979438080 bytes 2026-02-22T21:56:28.791 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22300 IOs, 10008925184 bytes 2026-02-22T21:56:28.828 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22400 IOs, 10047836160 bytes 2026-02-22T21:56:28.876 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22500 IOs, 10064359424 bytes 2026-02-22T21:56:28.938 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22600 IOs, 10130731008 bytes 2026-02-22T21:56:29.001 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22700 IOs, 10204129280 bytes 2026-02-22T21:56:29.240 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22800 IOs, 10212134912 bytes 2026-02-22T21:56:29.268 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22900 IOs, 10212714496 bytes 2026-02-22T21:56:29.309 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23000 IOs, 10234303488 bytes 2026-02-22T21:56:29.344 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23100 IOs, 10244997120 bytes 2026-02-22T21:56:29.390 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23200 IOs, 10281178112 bytes 2026-02-22T21:56:29.451 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23300 IOs, 10331729920 bytes 2026-02-22T21:56:29.505 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23400 IOs, 10375224320 bytes 2026-02-22T21:56:29.541 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23500 IOs, 10385257472 bytes 2026-02-22T21:56:29.622 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23600 IOs, 10492251136 bytes 2026-02-22T21:56:29.692 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23700 IOs, 10572367872 bytes 2026-02-22T21:56:29.743 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23800 IOs, 10622827520 bytes 2026-02-22T21:56:29.791 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23900 IOs, 10653670400 bytes 2026-02-22T21:56:29.842 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24000 IOs, 10686439424 bytes 2026-02-22T21:56:29.894 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24100 IOs, 10728768512 bytes 2026-02-22T21:56:29.968 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24200 IOs, 10815613952 bytes 2026-02-22T21:56:30.029 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24300 IOs, 10872996864 bytes 2026-02-22T21:56:30.088 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24400 IOs, 10933535744 bytes 2026-02-22T21:56:30.120 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24500 IOs, 10941187072 bytes 2026-02-22T21:56:30.158 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24600 IOs, 10980941824 bytes 2026-02-22T21:56:30.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24700 IOs, 11003430912 bytes 2026-02-22T21:56:30.245 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24800 IOs, 11018724352 bytes 2026-02-22T21:56:30.309 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24900 IOs, 11102501888 bytes 2026-02-22T21:56:30.364 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25000 IOs, 11144700928 bytes 2026-02-22T21:56:30.410 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25100 IOs, 11192829952 bytes 2026-02-22T21:56:30.450 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25200 IOs, 11210483712 bytes 2026-02-22T21:56:30.483 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25300 IOs, 11211120640 bytes 2026-02-22T21:56:30.527 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25400 IOs, 11240517632 bytes 2026-02-22T21:56:30.563 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25500 IOs, 11268899840 bytes 2026-02-22T21:56:30.642 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25600 IOs, 11351151616 bytes 2026-02-22T21:56:30.697 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25700 IOs, 11386186752 bytes 2026-02-22T21:56:30.729 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25800 IOs, 11390862336 bytes 2026-02-22T21:56:30.782 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25900 IOs, 11435831296 bytes 2026-02-22T21:56:30.862 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:26000 IOs, 11537030144 bytes 2026-02-22T21:56:30.893 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:26100 IOs, 11537653760 bytes 2026-02-22T21:56:30.921 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:26200 IOs, 11540552704 bytes 2026-02-22T21:56:30.991 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:26300 IOs, 11620386816 bytes 2026-02-22T21:56:31.034 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:26400 IOs, 11641571328 bytes 2026-02-22T21:56:31.049 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.4912370051502105) > weight (0.25) 2026-02-22T21:56:31.049 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.31266999062323264) > weight (0.25) 2026-02-22T21:56:31.049 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.632189711152721) > weight (0.25) 2026-02-22T21:56:31.049 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.3 with skip (0.8642978566219484) > weight (0.25) 2026-02-22T21:56:31.049 INFO:tasks.rbd_mirror_thrash:waiting for 95.04009028136943 secs before thrashing 2026-02-22T21:56:31.083 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:26500 IOs, 11676784640 bytes 2026-02-22T21:56:31.138 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:26600 IOs, 11740263424 bytes 2026-02-22T21:56:31.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:26700 IOs, 11772592128 bytes 2026-02-22T21:56:31.226 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ error_code=124 2026-02-22T21:56:31.226 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set -e 2026-02-22T21:56:31.226 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' 124 -eq 124 ']' 2026-02-22T21:56:31.226 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:56:31.226 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_pool_images cluster2 mirror 50 2026-02-22T21:56:31.226 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:56:31.226 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:56:31.226 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_count=50 2026-02-22T21:56:31.226 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T21:56:31.226 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local count 2026-02-22T21:56:31.226 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local last_count=0 2026-02-22T21:56:31.226 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:56:31.226 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ seq 1 40 2026-02-22T21:56:31.227 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in `seq 1 40` 2026-02-22T21:56:31.227 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 1 -ne 1 2026-02-22T21:56:31.227 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster cluster2 -p mirror mirror pool status 2026-02-22T21:56:31.227 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ grep 'images: ' 2026-02-22T21:56:31.227 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cut '-d ' -f 2 2026-02-22T21:56:31.251 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: failed to query services: (13) Permission denied 2026-02-22T21:56:31.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ count=50 2026-02-22T21:56:31.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 50 = 50 2026-02-22T21:56:31.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:56:31.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_pool_healthy cluster2 mirror 2026-02-22T21:56:31.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:56:31.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:56:31.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T21:56:31.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state 2026-02-22T21:56:31.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ seq 1 40 2026-02-22T21:56:31.254 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in `seq 1 40` 2026-02-22T21:56:31.254 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 1 -ne 1 2026-02-22T21:56:31.254 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster cluster2 -p mirror mirror pool status 2026-02-22T21:56:31.255 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ grep 'image health:' 2026-02-22T21:56:31.255 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cut '-d ' -f 3 2026-02-22T21:56:31.276 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: failed to query services: (13) Permission denied 2026-02-22T21:56:31.277 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ state=OK 2026-02-22T21:56:31.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test OK = ERROR 2026-02-22T21:56:31.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test OK = OK 2026-02-22T21:56:31.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:56:31.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_pool_images cluster1 mirror 50 2026-02-22T21:56:31.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:56:31.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:56:31.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_count=50 2026-02-22T21:56:31.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T21:56:31.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local count 2026-02-22T21:56:31.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local last_count=0 2026-02-22T21:56:31.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:56:31.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ seq 1 40 2026-02-22T21:56:31.279 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in `seq 1 40` 2026-02-22T21:56:31.279 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 1 -ne 1 2026-02-22T21:56:31.279 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster cluster1 -p mirror mirror pool status 2026-02-22T21:56:31.279 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ grep 'images: ' 2026-02-22T21:56:31.279 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cut '-d ' -f 2 2026-02-22T21:56:31.303 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: failed to query services: (13) Permission denied 2026-02-22T21:56:31.305 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ count=50 2026-02-22T21:56:31.305 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 50 = 50 2026-02-22T21:56:31.305 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:56:31.305 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_pool_healthy cluster1 mirror 2026-02-22T21:56:31.305 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:56:31.305 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:56:31.305 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T21:56:31.305 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state 2026-02-22T21:56:31.305 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ seq 1 40 2026-02-22T21:56:31.306 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in `seq 1 40` 2026-02-22T21:56:31.306 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 1 -ne 1 2026-02-22T21:56:31.307 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster cluster1 -p mirror mirror pool status 2026-02-22T21:56:31.307 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ grep 'image health:' 2026-02-22T21:56:31.307 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cut '-d ' -f 3 2026-02-22T21:56:31.331 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: failed to query services: (13) Permission denied 2026-02-22T21:56:31.332 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ state=WARNING 2026-02-22T21:56:31.332 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test WARNING = ERROR 2026-02-22T21:56:31.332 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test WARNING = OK 2026-02-22T21:56:31.332 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in `seq 1 40` 2026-02-22T21:56:31.332 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 2 -ne 1 2026-02-22T21:56:31.332 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 30 2026-02-22T21:57:01.335 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster cluster1 -p mirror mirror pool status 2026-02-22T21:57:01.335 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ grep 'image health:' 2026-02-22T21:57:01.335 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cut '-d ' -f 3 2026-02-22T21:57:01.361 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: failed to query services: (13) Permission denied 2026-02-22T21:57:01.363 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ state=OK 2026-02-22T21:57:01.363 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test OK = ERROR 2026-02-22T21:57:01.363 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test OK = OK 2026-02-22T21:57:01.363 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:57:01.363 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ testlog 'TEST: compare many images' 2026-02-22T21:57:01.364 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ tee -a /tmp/tmp.j44LZPowal/rbd-mirror.test.log 2026-02-22T21:57:01.364 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ date '+%F %T' 2026-02-22T21:57:01.364 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ echo 2026-02-22 21:57:01 TEST: compare many images 2026-02-22T21:57:01.364 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:2026-02-22 21:57:01 TEST: compare many images 2026-02-22T21:57:01.365 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ seq 1 50 2026-02-22T21:57:01.365 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T21:57:01.366 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_1 2026-02-22T21:57:01.366 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_snap cluster2 mirror image_1 snap 2026-02-22T21:57:01.366 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:57:01.366 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:01.366 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_1 2026-02-22T21:57:01.366 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T21:57:01.366 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror snap create image_1@snap --debug-rbd=20 --debug-journaler=20 2026-02-22T21:57:03.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_started cluster1 mirror image_1 2026-02-22T21:57:03.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:57:03.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:03.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_1 2026-02-22T21:57:03.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_state cluster1 mirror image_1 started 2026-02-22T21:57:03.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:57:03.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:03.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_1 2026-02-22T21:57:03.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=started 2026-02-22T21:57:03.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T21:57:03.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:57:03.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T21:57:04.183 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_replay_state cluster1 mirror image_1 started 2026-02-22T21:57:04.184 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:57:04.184 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:04.184 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_1 2026-02-22T21:57:04.184 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=started 2026-02-22T21:57:04.184 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_result 2026-02-22T21:57:04.184 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=stopped 2026-02-22T21:57:04.184 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_1 2026-02-22T21:57:04.184 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_instance=cluster1 2026-02-22T21:57:04.184 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ grep -i state 2026-02-22T21:57:04.185 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:57:04.185 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T21:57:04.185 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance=cluster1 2026-02-22T21:57:04.185 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local loop_instance 2026-02-22T21:57:04.185 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:57:04.185 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sleep 0 2026-02-22T21:57:04.188 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:57:04.188 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ seq 0 3 2026-02-22T21:57:04.188 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:57:04.188 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_1 2026-02-22T21:57:04.188 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:0 2026-02-22T21:57:04.188 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:57:04.188 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:57:04.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:57:04.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:0 2026-02-22T21:57:04.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:57:04.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:57:04.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:57:04.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=0 2026-02-22T21:57:04.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:57:04.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:57:04.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:57:04.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=0 2026-02-22T21:57:04.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T21:57:04.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:57:04.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:57:04.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:57:04.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:57:04.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:57:04.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:0 2026-02-22T21:57:04.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:57:04.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:57:04.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:57:04.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T21:57:04.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:57:04.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:57:04.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:57:04.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=0 2026-02-22T21:57:04.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=0 2026-02-22T21:57:04.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:57:04.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:57:04.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:57:04.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:57:04.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_1 2026-02-22T21:57:04.244 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:no valid command found; 2 closest matches: 2026-02-22T21:57:04.244 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status mirror/image_13 2026-02-22T21:57:04.244 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status mirror/image_17 2026-02-22T21:57:04.244 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T21:57:04.246 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:57:04.246 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_1 2026-02-22T21:57:04.246 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:1 2026-02-22T21:57:04.246 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:57:04.246 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:57:04.246 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T21:57:04.246 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:1 2026-02-22T21:57:04.246 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:57:04.246 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:57:04.246 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:57:04.246 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=1 2026-02-22T21:57:04.246 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T21:57:04.246 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:57:04.246 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:57:04.246 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=1 2026-02-22T21:57:04.246 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T21:57:04.246 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T21:57:04.246 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:1 2026-02-22T21:57:04.246 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:57:04.246 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:57:04.246 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T21:57:04.246 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:1 2026-02-22T21:57:04.246 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:57:04.246 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:57:04.246 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:57:04.247 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T21:57:04.247 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T21:57:04.247 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:57:04.247 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:57:04.247 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=1 2026-02-22T21:57:04.247 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=1 2026-02-22T21:57:04.247 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T21:57:04.258 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:57:04.258 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:57:04.259 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:57:04.259 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_1 2026-02-22T21:57:04.309 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ return 0 2026-02-22T21:57:04.309 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:57:04.309 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:57:04.310 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep -i Replaying 2026-02-22T21:57:04.311 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: "state": "Replaying" 2026-02-22T21:57:04.311 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_state=started 2026-02-22T21:57:04.311 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test started = started 2026-02-22T21:57:04.311 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:57:04.311 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_1 2026-02-22T21:57:04.311 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T21:57:04.311 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:57:04.311 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:04.311 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_1 2026-02-22T21:57:04.311 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = journal ']' 2026-02-22T21:57:04.311 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T21:57:04.311 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ mirror_image_snapshot cluster2 mirror image_1 2026-02-22T21:57:04.311 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:57:04.311 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:04.311 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_1 2026-02-22T21:57:04.311 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_1 2026-02-22T21:57:05.184 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Snapshot ID: 86 2026-02-22T21:57:05.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_1 2026-02-22T21:57:05.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T21:57:05.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:57:05.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:05.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_1 2026-02-22T21:57:05.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster2-mirror-image_1.status 2026-02-22T21:57:05.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster2-mirror-image_1.status 2026-02-22T21:57:05.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T21:57:05.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_log=/tmp/tmp.j44LZPowal/cluster2-mirror-image_1.status 2026-02-22T21:57:05.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster1-mirror-image_1.status 2026-02-22T21:57:05.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster1-mirror-image_1.status 2026-02-22T21:57:05.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T21:57:05.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_status_log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_1.status 2026-02-22T21:57:05.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_1 /tmp/tmp.j44LZPowal/cluster2-mirror-image_1.status 2026-02-22T21:57:05.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:57:05.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:05.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_1 2026-02-22T21:57:05.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster2-mirror-image_1.status 2026-02-22T21:57:05.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_1 --format xml 2026-02-22T21:57:05.195 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:57:05.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T21:57:05.223 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snapshot_id=86 2026-02-22T21:57:05.223 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:57:05.223 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:57:05.223 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.2 2026-02-22T21:57:05.425 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_1 /tmp/tmp.j44LZPowal/cluster1-mirror-image_1.status 2026-02-22T21:57:05.425 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:57:05.425 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:05.425 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_1 2026-02-22T21:57:05.425 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_1.status 2026-02-22T21:57:05.425 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_1 --format xml 2026-02-22T21:57:05.425 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:57:05.456 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:57:05.458 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id=35 2026-02-22T21:57:05.458 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 86 = 35 2026-02-22T21:57:05.458 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:57:05.458 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.4 2026-02-22T21:57:05.859 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_1 /tmp/tmp.j44LZPowal/cluster1-mirror-image_1.status 2026-02-22T21:57:05.859 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:57:05.859 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:05.859 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_1 2026-02-22T21:57:05.859 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_1.status 2026-02-22T21:57:05.859 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_1 --format xml 2026-02-22T21:57:05.859 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:57:05.889 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:57:05.889 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:57:05.890 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T21:57:05.890 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T21:57:05.890 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T21:57:05.891 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T21:57:05.891 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 86 = '' 2026-02-22T21:57:05.891 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:57:05.891 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.8 2026-02-22T21:57:06.692 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_1 /tmp/tmp.j44LZPowal/cluster1-mirror-image_1.status 2026-02-22T21:57:06.692 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:57:06.692 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:06.692 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_1 2026-02-22T21:57:06.692 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_1.status 2026-02-22T21:57:06.692 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_1 --format xml 2026-02-22T21:57:06.692 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:57:06.724 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:57:06.726 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id=86 2026-02-22T21:57:06.727 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 86 = 86 2026-02-22T21:57:06.727 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:57:06.727 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snap_present cluster1 mirror image_1 snap 2026-02-22T21:57:06.727 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:57:06.727 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:06.728 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_1 2026-02-22T21:57:06.728 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T21:57:06.728 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T21:57:06.728 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T21:57:06.728 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T21:57:07.726 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 info mirror/image_1@snap 2026-02-22T21:57:07.753 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_1': 2026-02-22T21:57:07.753 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 128 MiB in 32 objects 2026-02-22T21:57:07.753 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T21:57:07.753 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 2 2026-02-22T21:57:07.753 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 1109955f6f7f 2026-02-22T21:57:07.753 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.1109955f6f7f 2026-02-22T21:57:07.753 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T21:57:07.753 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock, non-primary 2026-02-22T21:57:07.754 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T21:57:07.754 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T21:57:07.754 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:47:59 2026 2026-02-22T21:57:07.754 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:47:59 2026 2026-02-22T21:57:07.754 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:47:59 2026 2026-02-22T21:57:07.754 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: protected: False 2026-02-22T21:57:07.754 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T21:57:07.754 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T21:57:07.754 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: 90c63745-c08b-46eb-b4e8-44a4f1e91721 2026-02-22T21:57:07.754 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: false 2026-02-22T21:57:07.755 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:57:07.756 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ compare_image_snaps mirror image_1 snap 2026-02-22T21:57:07.756 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:07.756 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_1 2026-02-22T21:57:07.756 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T21:57:07.756 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local ret=0 2026-02-22T21:57:07.756 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local rmt_export=/tmp/tmp.j44LZPowal/cluster2-mirror-image_1.export 2026-02-22T21:57:07.756 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local loc_export=/tmp/tmp.j44LZPowal/cluster1-mirror-image_1.export 2026-02-22T21:57:07.756 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-image_1.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_1.export 2026-02-22T21:57:07.756 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror export image_1@snap /tmp/tmp.j44LZPowal/cluster2-mirror-image_1.export 2026-02-22T21:57:07.862 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T21:57:07.869 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 -p mirror export image_1@snap /tmp/tmp.j44LZPowal/cluster1-mirror-image_1.export 2026-02-22T21:57:07.994 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T21:57:08.002 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ cmp /tmp/tmp.j44LZPowal/cluster2-mirror-image_1.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_1.export 2026-02-22T21:57:08.049 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-image_1.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_1.export 2026-02-22T21:57:08.083 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:57:08.083 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T21:57:08.083 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_2 2026-02-22T21:57:08.083 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_snap cluster2 mirror image_2 snap 2026-02-22T21:57:08.083 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:57:08.083 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:08.083 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_2 2026-02-22T21:57:08.083 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T21:57:08.083 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror snap create image_2@snap --debug-rbd=20 --debug-journaler=20 2026-02-22T21:57:09.188 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_started cluster1 mirror image_2 2026-02-22T21:57:09.188 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:57:09.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:09.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_2 2026-02-22T21:57:09.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_state cluster1 mirror image_2 started 2026-02-22T21:57:09.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:57:09.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:09.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_2 2026-02-22T21:57:09.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=started 2026-02-22T21:57:09.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T21:57:09.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:57:09.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T21:57:10.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_replay_state cluster1 mirror image_2 started 2026-02-22T21:57:10.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:57:10.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:10.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_2 2026-02-22T21:57:10.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=started 2026-02-22T21:57:10.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_result 2026-02-22T21:57:10.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=stopped 2026-02-22T21:57:10.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_2 2026-02-22T21:57:10.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_instance=cluster1 2026-02-22T21:57:10.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:57:10.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T21:57:10.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance=cluster1 2026-02-22T21:57:10.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local loop_instance 2026-02-22T21:57:10.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:57:10.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sleep 0 2026-02-22T21:57:10.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ grep -i state 2026-02-22T21:57:10.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:57:10.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ seq 0 3 2026-02-22T21:57:10.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:57:10.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_2 2026-02-22T21:57:10.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:0 2026-02-22T21:57:10.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:57:10.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:57:10.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:57:10.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:0 2026-02-22T21:57:10.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:57:10.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:57:10.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:57:10.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=0 2026-02-22T21:57:10.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:57:10.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:57:10.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:57:10.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=0 2026-02-22T21:57:10.195 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T21:57:10.195 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:57:10.195 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:57:10.195 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:57:10.195 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:57:10.195 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:57:10.195 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:0 2026-02-22T21:57:10.195 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:57:10.195 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:57:10.195 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:57:10.195 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T21:57:10.195 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:57:10.195 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:57:10.195 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:57:10.195 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=0 2026-02-22T21:57:10.195 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=0 2026-02-22T21:57:10.195 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:57:10.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:57:10.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:57:10.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:57:10.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_2 2026-02-22T21:57:10.252 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:no valid command found; 5 closest matches: 2026-02-22T21:57:10.252 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status mirror/image_21 2026-02-22T21:57:10.252 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status mirror/image_22 2026-02-22T21:57:10.252 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status mirror/image_25 2026-02-22T21:57:10.252 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status mirror/image_28 2026-02-22T21:57:10.252 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status mirror/image_29 2026-02-22T21:57:10.252 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T21:57:10.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:57:10.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_2 2026-02-22T21:57:10.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:1 2026-02-22T21:57:10.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:57:10.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:57:10.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T21:57:10.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:1 2026-02-22T21:57:10.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:57:10.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:57:10.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:57:10.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=1 2026-02-22T21:57:10.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T21:57:10.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:57:10.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:57:10.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=1 2026-02-22T21:57:10.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T21:57:10.254 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T21:57:10.254 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:1 2026-02-22T21:57:10.254 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:57:10.254 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:57:10.254 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T21:57:10.254 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:1 2026-02-22T21:57:10.254 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:57:10.254 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:57:10.256 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:57:10.257 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T21:57:10.257 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T21:57:10.257 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:57:10.257 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:57:10.257 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=1 2026-02-22T21:57:10.257 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=1 2026-02-22T21:57:10.257 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T21:57:10.265 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:57:10.265 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:57:10.265 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:57:10.265 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_2 2026-02-22T21:57:10.312 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:no valid command found; 2 closest matches: 2026-02-22T21:57:10.312 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status mirror/image_23 2026-02-22T21:57:10.312 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status mirror/image_27 2026-02-22T21:57:10.312 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T21:57:10.313 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:57:10.313 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_2 2026-02-22T21:57:10.313 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:2 2026-02-22T21:57:10.313 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:57:10.313 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:57:10.313 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T21:57:10.313 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:2 2026-02-22T21:57:10.313 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:57:10.313 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:57:10.313 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:57:10.313 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=2 2026-02-22T21:57:10.313 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T21:57:10.313 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:57:10.313 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:57:10.313 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=2 2026-02-22T21:57:10.313 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T21:57:10.314 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T21:57:10.314 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:2 2026-02-22T21:57:10.314 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:57:10.314 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:57:10.314 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T21:57:10.314 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:2 2026-02-22T21:57:10.314 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:57:10.314 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:57:10.314 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:57:10.314 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T21:57:10.314 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T21:57:10.314 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:57:10.314 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:57:10.314 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=2 2026-02-22T21:57:10.314 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=2 2026-02-22T21:57:10.314 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T21:57:10.325 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:57:10.325 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:57:10.325 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:57:10.325 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_2 2026-02-22T21:57:10.370 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:no valid command found; 2 closest matches: 2026-02-22T21:57:10.370 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status mirror/image_20 2026-02-22T21:57:10.370 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status mirror/image_26 2026-02-22T21:57:10.370 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T21:57:10.372 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:57:10.372 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_2 2026-02-22T21:57:10.372 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:3 2026-02-22T21:57:10.372 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:57:10.372 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:57:10.372 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T21:57:10.372 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:3 2026-02-22T21:57:10.372 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:57:10.372 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:57:10.372 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:57:10.372 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=3 2026-02-22T21:57:10.372 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T21:57:10.372 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:57:10.372 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:57:10.372 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=3 2026-02-22T21:57:10.372 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=3 2026-02-22T21:57:10.372 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T21:57:10.372 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:3 2026-02-22T21:57:10.372 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:57:10.372 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:57:10.372 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T21:57:10.372 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:3 2026-02-22T21:57:10.372 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:57:10.372 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:57:10.372 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:57:10.372 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=3 2026-02-22T21:57:10.373 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T21:57:10.373 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:57:10.373 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:57:10.373 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=3 2026-02-22T21:57:10.373 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=3 2026-02-22T21:57:10.373 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T21:57:10.383 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:57:10.383 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:57:10.384 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:57:10.384 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_2 2026-02-22T21:57:10.432 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ return 0 2026-02-22T21:57:10.433 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:57:10.433 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:57:10.433 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep -i Replaying 2026-02-22T21:57:10.433 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: "state": "Replaying" 2026-02-22T21:57:10.435 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_state=started 2026-02-22T21:57:10.435 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test started = started 2026-02-22T21:57:10.435 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:57:10.435 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_2 2026-02-22T21:57:10.435 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T21:57:10.435 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:57:10.435 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:10.435 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_2 2026-02-22T21:57:10.436 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = journal ']' 2026-02-22T21:57:10.436 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T21:57:10.436 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ mirror_image_snapshot cluster2 mirror image_2 2026-02-22T21:57:10.436 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:57:10.436 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:10.436 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_2 2026-02-22T21:57:10.436 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_2 2026-02-22T21:57:11.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Snapshot ID: 88 2026-02-22T21:57:11.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_2 2026-02-22T21:57:11.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T21:57:11.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:57:11.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:11.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_2 2026-02-22T21:57:11.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster2-mirror-image_2.status 2026-02-22T21:57:11.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster2-mirror-image_2.status 2026-02-22T21:57:11.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T21:57:11.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_log=/tmp/tmp.j44LZPowal/cluster2-mirror-image_2.status 2026-02-22T21:57:11.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster1-mirror-image_2.status 2026-02-22T21:57:11.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster1-mirror-image_2.status 2026-02-22T21:57:11.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T21:57:11.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_status_log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_2.status 2026-02-22T21:57:11.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_2 /tmp/tmp.j44LZPowal/cluster2-mirror-image_2.status 2026-02-22T21:57:11.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:57:11.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:11.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_2 2026-02-22T21:57:11.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster2-mirror-image_2.status 2026-02-22T21:57:11.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_2 --format xml 2026-02-22T21:57:11.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:57:11.230 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T21:57:11.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snapshot_id=88 2026-02-22T21:57:11.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:57:11.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:57:11.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.2 2026-02-22T21:57:11.434 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_2 /tmp/tmp.j44LZPowal/cluster1-mirror-image_2.status 2026-02-22T21:57:11.434 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:57:11.434 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:11.434 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_2 2026-02-22T21:57:11.434 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_2.status 2026-02-22T21:57:11.434 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_2 --format xml 2026-02-22T21:57:11.435 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:57:11.463 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:57:11.465 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id=36 2026-02-22T21:57:11.465 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 88 = 36 2026-02-22T21:57:11.465 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:57:11.465 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.4 2026-02-22T21:57:11.865 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_2 /tmp/tmp.j44LZPowal/cluster1-mirror-image_2.status 2026-02-22T21:57:11.865 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:57:11.865 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:11.866 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_2 2026-02-22T21:57:11.866 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_2.status 2026-02-22T21:57:11.866 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_2 --format xml 2026-02-22T21:57:11.866 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:57:11.896 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:57:11.896 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:57:11.897 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T21:57:11.897 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T21:57:11.897 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T21:57:11.897 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T21:57:11.897 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 88 = '' 2026-02-22T21:57:11.897 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:57:11.898 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.8 2026-02-22T21:57:12.699 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_2 /tmp/tmp.j44LZPowal/cluster1-mirror-image_2.status 2026-02-22T21:57:12.699 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:57:12.699 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:12.699 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_2 2026-02-22T21:57:12.699 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_2.status 2026-02-22T21:57:12.699 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_2 --format xml 2026-02-22T21:57:12.699 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:57:12.730 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:57:12.732 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id=88 2026-02-22T21:57:12.732 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 88 = 88 2026-02-22T21:57:12.732 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:57:12.732 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snap_present cluster1 mirror image_2 snap 2026-02-22T21:57:12.732 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:57:12.732 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:12.732 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_2 2026-02-22T21:57:12.733 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T21:57:12.733 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T21:57:12.733 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T21:57:12.733 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T21:57:13.733 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 info mirror/image_2@snap 2026-02-22T21:57:13.762 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_2': 2026-02-22T21:57:13.762 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 128 MiB in 32 objects 2026-02-22T21:57:13.762 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T21:57:13.762 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 2 2026-02-22T21:57:13.763 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 119a6c005c20 2026-02-22T21:57:13.763 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.119a6c005c20 2026-02-22T21:57:13.763 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T21:57:13.763 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock, non-primary 2026-02-22T21:57:13.763 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T21:57:13.763 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T21:57:13.763 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:48:11 2026 2026-02-22T21:57:13.763 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:48:11 2026 2026-02-22T21:57:13.763 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:48:11 2026 2026-02-22T21:57:13.763 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: protected: False 2026-02-22T21:57:13.763 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T21:57:13.763 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T21:57:13.764 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: 9e277158-7fce-48dd-8563-7bdcdfac29d0 2026-02-22T21:57:13.764 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: false 2026-02-22T21:57:13.765 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:57:13.765 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ compare_image_snaps mirror image_2 snap 2026-02-22T21:57:13.765 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:13.765 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_2 2026-02-22T21:57:13.765 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T21:57:13.765 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local ret=0 2026-02-22T21:57:13.765 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local rmt_export=/tmp/tmp.j44LZPowal/cluster2-mirror-image_2.export 2026-02-22T21:57:13.766 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local loc_export=/tmp/tmp.j44LZPowal/cluster1-mirror-image_2.export 2026-02-22T21:57:13.766 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-image_2.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_2.export 2026-02-22T21:57:13.766 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror export image_2@snap /tmp/tmp.j44LZPowal/cluster2-mirror-image_2.export 2026-02-22T21:57:13.870 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T21:57:13.877 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 -p mirror export image_2@snap /tmp/tmp.j44LZPowal/cluster1-mirror-image_2.export 2026-02-22T21:57:13.994 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T21:57:14.001 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ cmp /tmp/tmp.j44LZPowal/cluster2-mirror-image_2.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_2.export 2026-02-22T21:57:14.049 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-image_2.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_2.export 2026-02-22T21:57:14.082 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:57:14.082 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T21:57:14.082 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_3 2026-02-22T21:57:14.082 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_snap cluster2 mirror image_3 snap 2026-02-22T21:57:14.082 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:57:14.082 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:14.082 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_3 2026-02-22T21:57:14.082 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T21:57:14.082 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror snap create image_3@snap --debug-rbd=20 --debug-journaler=20 2026-02-22T21:57:15.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_started cluster1 mirror image_3 2026-02-22T21:57:15.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:57:15.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:15.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_3 2026-02-22T21:57:15.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_state cluster1 mirror image_3 started 2026-02-22T21:57:15.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:57:15.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:15.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_3 2026-02-22T21:57:15.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=started 2026-02-22T21:57:15.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T21:57:15.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:57:15.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T21:57:15.301 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.8249922086374721) > weight (0.25) 2026-02-22T21:57:15.302 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.7712289355874392) > weight (0.25) 2026-02-22T21:57:15.302 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.41860246701818604) > weight (0.25) 2026-02-22T21:57:15.302 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.3 2026-02-22T21:57:15.302 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Sent signal 15 2026-02-22T21:57:15.302 INFO:tasks.rbd_mirror_thrash:waiting for 0.0012408037078448508 secs before reviving daemons 2026-02-22T21:57:15.303 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial026.stderr:2026-02-22T21:57:15.302+0000 7f5264676640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.3 (PID: 60154) UID: 1000 2026-02-22T21:57:15.304 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.3 2026-02-22T21:57:15.304 INFO:teuthology.orchestra.run:waiting for 600 2026-02-22T21:57:16.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_replay_state cluster1 mirror image_3 started 2026-02-22T21:57:16.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:57:16.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:16.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_3 2026-02-22T21:57:16.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=started 2026-02-22T21:57:16.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_result 2026-02-22T21:57:16.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=stopped 2026-02-22T21:57:16.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_3 2026-02-22T21:57:16.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_instance=cluster1 2026-02-22T21:57:16.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:57:16.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ grep -i state 2026-02-22T21:57:16.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T21:57:16.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance=cluster1 2026-02-22T21:57:16.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local loop_instance 2026-02-22T21:57:16.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:57:16.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sleep 0 2026-02-22T21:57:16.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:57:16.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ seq 0 3 2026-02-22T21:57:16.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:57:16.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_3 2026-02-22T21:57:16.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:0 2026-02-22T21:57:16.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:57:16.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:57:16.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:57:16.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:0 2026-02-22T21:57:16.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:57:16.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:57:16.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:57:16.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=0 2026-02-22T21:57:16.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:57:16.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:57:16.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:57:16.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=0 2026-02-22T21:57:16.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T21:57:16.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:57:16.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:57:16.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:57:16.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:57:16.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:57:16.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:0 2026-02-22T21:57:16.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:57:16.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:57:16.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:57:16.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T21:57:16.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:57:16.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:57:16.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:57:16.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=0 2026-02-22T21:57:16.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=0 2026-02-22T21:57:16.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:57:16.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:57:16.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:57:16.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:57:16.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_3 2026-02-22T21:57:16.263 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:no valid command found; 2 closest matches: 2026-02-22T21:57:16.263 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status mirror/image_33 2026-02-22T21:57:16.263 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status mirror/image_37 2026-02-22T21:57:16.263 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T21:57:16.268 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:57:16.268 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_3 2026-02-22T21:57:16.268 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:1 2026-02-22T21:57:16.269 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:57:16.269 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:57:16.269 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T21:57:16.269 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:1 2026-02-22T21:57:16.269 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:57:16.270 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:57:16.270 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:57:16.270 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=1 2026-02-22T21:57:16.270 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T21:57:16.270 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:57:16.270 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:57:16.270 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=1 2026-02-22T21:57:16.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T21:57:16.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T21:57:16.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:1 2026-02-22T21:57:16.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:57:16.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:57:16.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T21:57:16.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:1 2026-02-22T21:57:16.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:57:16.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:57:16.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:57:16.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T21:57:16.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T21:57:16.273 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:57:16.273 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:57:16.273 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=1 2026-02-22T21:57:16.273 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=1 2026-02-22T21:57:16.273 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T21:57:16.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:57:16.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:57:16.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:57:16.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_3 2026-02-22T21:57:16.324 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:no valid command found; 4 closest matches: 2026-02-22T21:57:16.324 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status mirror/image_30 2026-02-22T21:57:16.324 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status mirror/image_31 2026-02-22T21:57:16.324 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status mirror/image_38 2026-02-22T21:57:16.324 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status mirror/image_39 2026-02-22T21:57:16.324 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T21:57:16.326 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:57:16.326 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_3 2026-02-22T21:57:16.327 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:2 2026-02-22T21:57:16.327 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:57:16.327 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:57:16.327 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T21:57:16.327 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:2 2026-02-22T21:57:16.327 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:57:16.327 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:57:16.327 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:57:16.327 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=2 2026-02-22T21:57:16.327 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T21:57:16.327 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:57:16.327 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:57:16.327 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=2 2026-02-22T21:57:16.327 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T21:57:16.327 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T21:57:16.328 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:2 2026-02-22T21:57:16.328 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:57:16.328 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:57:16.328 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T21:57:16.328 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:2 2026-02-22T21:57:16.328 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:57:16.328 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:57:16.328 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:57:16.328 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T21:57:16.328 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T21:57:16.328 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:57:16.328 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:57:16.328 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=2 2026-02-22T21:57:16.328 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=2 2026-02-22T21:57:16.328 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T21:57:16.338 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:57:16.338 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:57:16.338 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:57:16.338 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_3 2026-02-22T21:57:16.382 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:no valid command found; 1 closest matches: 2026-02-22T21:57:16.383 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status mirror/image_32 2026-02-22T21:57:16.383 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T21:57:16.386 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:57:16.386 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_3 2026-02-22T21:57:16.386 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:3 2026-02-22T21:57:16.386 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:57:16.386 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:57:16.387 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T21:57:16.387 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:3 2026-02-22T21:57:16.387 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:57:16.387 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:57:16.387 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:57:16.387 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=3 2026-02-22T21:57:16.387 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T21:57:16.387 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:57:16.387 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:57:16.387 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=3 2026-02-22T21:57:16.387 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=3 2026-02-22T21:57:16.387 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T21:57:16.388 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:3 2026-02-22T21:57:16.388 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:57:16.388 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:57:16.388 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T21:57:16.388 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:3 2026-02-22T21:57:16.388 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:57:16.388 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:57:16.388 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:57:16.388 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=3 2026-02-22T21:57:16.388 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T21:57:16.388 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:57:16.388 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:57:16.389 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=3 2026-02-22T21:57:16.389 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=3 2026-02-22T21:57:16.389 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T21:57:16.396 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:57:16.396 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:57:16.396 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:57:16.396 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_3 2026-02-22T21:57:16.444 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ return 0 2026-02-22T21:57:16.445 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:57:16.445 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:57:16.445 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep -i Replaying 2026-02-22T21:57:16.446 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: "state": "Replaying" 2026-02-22T21:57:16.446 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_state=started 2026-02-22T21:57:16.446 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test started = started 2026-02-22T21:57:16.446 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:57:16.446 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_3 2026-02-22T21:57:16.446 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T21:57:16.447 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:57:16.447 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:16.447 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_3 2026-02-22T21:57:16.447 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = journal ']' 2026-02-22T21:57:16.447 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T21:57:16.447 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ mirror_image_snapshot cluster2 mirror image_3 2026-02-22T21:57:16.447 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:57:16.447 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:16.447 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_3 2026-02-22T21:57:16.447 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_3 2026-02-22T21:57:17.401 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Snapshot ID: 90 2026-02-22T21:57:17.407 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_3 2026-02-22T21:57:17.407 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T21:57:17.407 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:57:17.408 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:17.408 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_3 2026-02-22T21:57:17.408 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster2-mirror-image_3.status 2026-02-22T21:57:17.408 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster2-mirror-image_3.status 2026-02-22T21:57:17.408 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T21:57:17.409 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_log=/tmp/tmp.j44LZPowal/cluster2-mirror-image_3.status 2026-02-22T21:57:17.409 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster1-mirror-image_3.status 2026-02-22T21:57:17.409 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster1-mirror-image_3.status 2026-02-22T21:57:17.409 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T21:57:17.411 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_status_log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_3.status 2026-02-22T21:57:17.411 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_3 /tmp/tmp.j44LZPowal/cluster2-mirror-image_3.status 2026-02-22T21:57:17.411 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:57:17.411 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:17.411 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_3 2026-02-22T21:57:17.411 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster2-mirror-image_3.status 2026-02-22T21:57:17.411 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_3 --format xml 2026-02-22T21:57:17.411 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:57:17.439 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T21:57:17.440 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snapshot_id=90 2026-02-22T21:57:17.441 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:57:17.441 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:57:17.441 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.2 2026-02-22T21:57:17.642 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_3 /tmp/tmp.j44LZPowal/cluster1-mirror-image_3.status 2026-02-22T21:57:17.642 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:57:17.642 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:17.642 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_3 2026-02-22T21:57:17.642 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_3.status 2026-02-22T21:57:17.642 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_3 --format xml 2026-02-22T21:57:17.642 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:57:17.672 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:57:17.672 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:57:17.673 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T21:57:17.673 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T21:57:17.674 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T21:57:17.674 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T21:57:17.674 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 90 = '' 2026-02-22T21:57:17.675 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:57:17.675 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.4 2026-02-22T21:57:18.075 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_3 /tmp/tmp.j44LZPowal/cluster1-mirror-image_3.status 2026-02-22T21:57:18.076 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:57:18.076 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:18.076 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_3 2026-02-22T21:57:18.076 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_3.status 2026-02-22T21:57:18.076 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_3 --format xml 2026-02-22T21:57:18.076 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:57:18.106 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:57:18.106 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:57:18.106 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T21:57:18.107 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T21:57:18.107 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T21:57:18.108 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T21:57:18.108 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 90 = '' 2026-02-22T21:57:18.108 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:57:18.108 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.8 2026-02-22T21:57:18.909 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_3 /tmp/tmp.j44LZPowal/cluster1-mirror-image_3.status 2026-02-22T21:57:18.909 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:57:18.909 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:18.909 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_3 2026-02-22T21:57:18.909 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_3.status 2026-02-22T21:57:18.909 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_3 --format xml 2026-02-22T21:57:18.909 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:57:18.943 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:57:18.945 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id=90 2026-02-22T21:57:18.945 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 90 = 90 2026-02-22T21:57:18.946 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:57:18.946 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snap_present cluster1 mirror image_3 snap 2026-02-22T21:57:18.946 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:57:18.946 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:18.946 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_3 2026-02-22T21:57:18.947 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T21:57:18.947 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T21:57:18.947 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T21:57:18.947 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T21:57:19.945 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 info mirror/image_3@snap 2026-02-22T21:57:19.975 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_3': 2026-02-22T21:57:19.975 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 128 MiB in 32 objects 2026-02-22T21:57:19.976 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T21:57:19.976 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 2 2026-02-22T21:57:19.976 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 11bdd81f27a 2026-02-22T21:57:19.976 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.11bdd81f27a 2026-02-22T21:57:19.976 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T21:57:19.976 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock, non-primary 2026-02-22T21:57:19.976 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T21:57:19.976 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T21:57:19.976 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:48:18 2026 2026-02-22T21:57:19.976 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:48:18 2026 2026-02-22T21:57:19.977 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:48:18 2026 2026-02-22T21:57:19.977 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: protected: False 2026-02-22T21:57:19.977 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T21:57:19.977 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T21:57:19.977 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: 06da309f-d597-43fe-8ebc-0e374ae46a50 2026-02-22T21:57:19.977 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: false 2026-02-22T21:57:19.978 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:57:19.978 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ compare_image_snaps mirror image_3 snap 2026-02-22T21:57:19.979 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:19.979 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_3 2026-02-22T21:57:19.979 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T21:57:19.979 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local ret=0 2026-02-22T21:57:19.979 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local rmt_export=/tmp/tmp.j44LZPowal/cluster2-mirror-image_3.export 2026-02-22T21:57:19.979 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local loc_export=/tmp/tmp.j44LZPowal/cluster1-mirror-image_3.export 2026-02-22T21:57:19.979 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-image_3.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_3.export 2026-02-22T21:57:19.979 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror export image_3@snap /tmp/tmp.j44LZPowal/cluster2-mirror-image_3.export 2026-02-22T21:57:20.090 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T21:57:20.098 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 -p mirror export image_3@snap /tmp/tmp.j44LZPowal/cluster1-mirror-image_3.export 2026-02-22T21:57:20.237 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T21:57:20.246 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ cmp /tmp/tmp.j44LZPowal/cluster2-mirror-image_3.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_3.export 2026-02-22T21:57:20.296 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-image_3.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_3.export 2026-02-22T21:57:20.331 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:57:20.331 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T21:57:20.331 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_4 2026-02-22T21:57:20.331 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_snap cluster2 mirror image_4 snap 2026-02-22T21:57:20.331 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:57:20.331 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:20.331 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_4 2026-02-22T21:57:20.331 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T21:57:20.332 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror snap create image_4@snap --debug-rbd=20 --debug-journaler=20 2026-02-22T21:57:21.406 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.3 2026-02-22T21:57:21.406 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Restarting daemon 2026-02-22T21:57:21.407 DEBUG:teuthology.orchestra.run.trial026:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.3 2026-02-22T21:57:21.409 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Started 2026-02-22T21:57:21.409 INFO:tasks.rbd_mirror_thrash:waiting for 69.14046584795601 secs before thrashing 2026-02-22T21:57:21.429 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_started cluster1 mirror image_4 2026-02-22T21:57:21.429 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:57:21.429 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:21.429 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_4 2026-02-22T21:57:21.430 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_state cluster1 mirror image_4 started 2026-02-22T21:57:21.430 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:57:21.430 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:21.430 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_4 2026-02-22T21:57:21.430 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=started 2026-02-22T21:57:21.430 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T21:57:21.430 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:57:21.430 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T21:57:22.437 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_replay_state cluster1 mirror image_4 started 2026-02-22T21:57:22.438 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:57:22.438 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:22.438 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_4 2026-02-22T21:57:22.438 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=started 2026-02-22T21:57:22.439 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_result 2026-02-22T21:57:22.439 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=stopped 2026-02-22T21:57:22.439 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_4 2026-02-22T21:57:22.439 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ grep -i state 2026-02-22T21:57:22.439 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_instance=cluster1 2026-02-22T21:57:22.439 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:57:22.440 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T21:57:22.440 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance=cluster1 2026-02-22T21:57:22.440 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local loop_instance 2026-02-22T21:57:22.440 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:57:22.440 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sleep 0 2026-02-22T21:57:22.440 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:57:22.441 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ seq 0 3 2026-02-22T21:57:22.441 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:57:22.441 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_4 2026-02-22T21:57:22.441 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:0 2026-02-22T21:57:22.441 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:57:22.441 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:57:22.442 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:57:22.442 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:0 2026-02-22T21:57:22.442 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:57:22.442 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:57:22.442 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:57:22.442 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=0 2026-02-22T21:57:22.443 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:57:22.443 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:57:22.443 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:57:22.443 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=0 2026-02-22T21:57:22.443 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T21:57:22.443 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:57:22.444 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:57:22.444 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:57:22.444 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:57:22.444 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:57:22.444 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:0 2026-02-22T21:57:22.444 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:57:22.444 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:57:22.445 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:57:22.445 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T21:57:22.445 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:57:22.445 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:57:22.445 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:57:22.445 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=0 2026-02-22T21:57:22.445 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=0 2026-02-22T21:57:22.446 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:57:22.448 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:57:22.448 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:57:22.448 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:57:22.448 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_4 2026-02-22T21:57:22.496 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:no valid command found; 3 closest matches: 2026-02-22T21:57:22.496 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status mirror/image_41 2026-02-22T21:57:22.496 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status mirror/image_45 2026-02-22T21:57:22.496 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status mirror/image_49 2026-02-22T21:57:22.496 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T21:57:22.499 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:57:22.499 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_4 2026-02-22T21:57:22.499 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:1 2026-02-22T21:57:22.500 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:57:22.500 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:57:22.500 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T21:57:22.500 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:1 2026-02-22T21:57:22.500 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:57:22.500 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:57:22.500 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:57:22.500 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=1 2026-02-22T21:57:22.500 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T21:57:22.500 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:57:22.500 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:57:22.501 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=1 2026-02-22T21:57:22.501 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T21:57:22.501 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T21:57:22.501 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:1 2026-02-22T21:57:22.501 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:57:22.501 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:57:22.501 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T21:57:22.501 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:1 2026-02-22T21:57:22.501 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:57:22.501 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:57:22.501 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:57:22.501 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T21:57:22.502 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T21:57:22.502 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:57:22.502 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:57:22.502 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=1 2026-02-22T21:57:22.502 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=1 2026-02-22T21:57:22.502 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T21:57:22.510 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:57:22.510 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:57:22.510 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:57:22.510 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_4 2026-02-22T21:57:22.555 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:no valid command found; 2 closest matches: 2026-02-22T21:57:22.555 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status mirror/image_42 2026-02-22T21:57:22.555 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status mirror/image_46 2026-02-22T21:57:22.555 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T21:57:22.558 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:57:22.558 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_4 2026-02-22T21:57:22.558 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:2 2026-02-22T21:57:22.558 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:57:22.558 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:57:22.558 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T21:57:22.558 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:2 2026-02-22T21:57:22.559 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:57:22.559 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:57:22.559 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:57:22.559 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=2 2026-02-22T21:57:22.559 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T21:57:22.559 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:57:22.559 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:57:22.559 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=2 2026-02-22T21:57:22.559 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T21:57:22.559 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T21:57:22.559 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:2 2026-02-22T21:57:22.560 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:57:22.560 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:57:22.560 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T21:57:22.560 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:2 2026-02-22T21:57:22.560 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:57:22.560 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:57:22.560 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:57:22.560 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T21:57:22.560 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T21:57:22.560 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:57:22.560 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:57:22.561 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=2 2026-02-22T21:57:22.561 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=2 2026-02-22T21:57:22.561 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T21:57:22.567 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:57:22.567 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:57:22.568 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:57:22.568 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_4 2026-02-22T21:57:22.611 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:no valid command found; 3 closest matches: 2026-02-22T21:57:22.611 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status mirror/image_43 2026-02-22T21:57:22.612 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status mirror/image_44 2026-02-22T21:57:22.612 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status mirror/image_48 2026-02-22T21:57:22.612 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T21:57:22.615 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:57:22.615 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_4 2026-02-22T21:57:22.615 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:3 2026-02-22T21:57:22.615 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:57:22.615 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:57:22.615 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T21:57:22.615 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:3 2026-02-22T21:57:22.616 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:57:22.616 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:57:22.616 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:57:22.616 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=3 2026-02-22T21:57:22.616 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T21:57:22.616 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:57:22.616 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:57:22.616 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=3 2026-02-22T21:57:22.616 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=3 2026-02-22T21:57:22.616 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T21:57:22.616 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:3 2026-02-22T21:57:22.616 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:57:22.617 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:57:22.617 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T21:57:22.617 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:3 2026-02-22T21:57:22.617 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:57:22.617 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:57:22.617 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:57:22.617 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=3 2026-02-22T21:57:22.617 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T21:57:22.617 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:57:22.617 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:57:22.617 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=3 2026-02-22T21:57:22.617 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=3 2026-02-22T21:57:22.618 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T21:57:22.625 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:57:22.625 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:57:22.625 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:57:22.625 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_4 2026-02-22T21:57:22.674 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ return 0 2026-02-22T21:57:22.675 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:57:22.675 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:57:22.676 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep -i Replaying 2026-02-22T21:57:22.676 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: "state": "Replaying" 2026-02-22T21:57:22.677 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_state=started 2026-02-22T21:57:22.678 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test started = started 2026-02-22T21:57:22.678 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:57:22.678 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_4 2026-02-22T21:57:22.678 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T21:57:22.678 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:57:22.678 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:22.678 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_4 2026-02-22T21:57:22.678 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = journal ']' 2026-02-22T21:57:22.678 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T21:57:22.678 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ mirror_image_snapshot cluster2 mirror image_4 2026-02-22T21:57:22.678 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:57:22.679 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:22.679 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_4 2026-02-22T21:57:22.679 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_4 2026-02-22T21:57:23.433 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Snapshot ID: 92 2026-02-22T21:57:23.440 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_4 2026-02-22T21:57:23.440 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T21:57:23.440 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:57:23.440 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:23.440 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_4 2026-02-22T21:57:23.440 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster2-mirror-image_4.status 2026-02-22T21:57:23.440 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster2-mirror-image_4.status 2026-02-22T21:57:23.440 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T21:57:23.442 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_log=/tmp/tmp.j44LZPowal/cluster2-mirror-image_4.status 2026-02-22T21:57:23.442 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster1-mirror-image_4.status 2026-02-22T21:57:23.442 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster1-mirror-image_4.status 2026-02-22T21:57:23.443 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T21:57:23.444 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_status_log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_4.status 2026-02-22T21:57:23.444 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_4 /tmp/tmp.j44LZPowal/cluster2-mirror-image_4.status 2026-02-22T21:57:23.444 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:57:23.444 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:23.445 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_4 2026-02-22T21:57:23.445 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster2-mirror-image_4.status 2026-02-22T21:57:23.445 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_4 --format xml 2026-02-22T21:57:23.445 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:57:23.472 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T21:57:23.473 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snapshot_id=92 2026-02-22T21:57:23.473 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:57:23.474 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:57:23.474 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.2 2026-02-22T21:57:23.675 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_4 /tmp/tmp.j44LZPowal/cluster1-mirror-image_4.status 2026-02-22T21:57:23.675 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:57:23.675 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:23.675 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_4 2026-02-22T21:57:23.675 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_4.status 2026-02-22T21:57:23.676 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_4 --format xml 2026-02-22T21:57:23.676 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:57:23.705 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:57:23.706 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id=38 2026-02-22T21:57:23.707 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 92 = 38 2026-02-22T21:57:23.707 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:57:23.707 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.4 2026-02-22T21:57:24.108 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_4 /tmp/tmp.j44LZPowal/cluster1-mirror-image_4.status 2026-02-22T21:57:24.108 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:57:24.108 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:24.108 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_4 2026-02-22T21:57:24.108 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_4.status 2026-02-22T21:57:24.108 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_4 --format xml 2026-02-22T21:57:24.108 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:57:24.136 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:57:24.137 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id=38 2026-02-22T21:57:24.138 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 92 = 38 2026-02-22T21:57:24.138 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:57:24.138 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.8 2026-02-22T21:57:24.939 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_4 /tmp/tmp.j44LZPowal/cluster1-mirror-image_4.status 2026-02-22T21:57:24.939 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:57:24.940 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:24.940 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_4 2026-02-22T21:57:24.940 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_4.status 2026-02-22T21:57:24.940 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_4 --format xml 2026-02-22T21:57:24.940 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:57:24.967 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:57:24.968 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:57:24.969 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T21:57:24.969 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T21:57:24.969 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T21:57:24.969 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T21:57:24.970 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 92 = '' 2026-02-22T21:57:24.970 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:57:24.970 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1.6 2026-02-22T21:57:26.570 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_4 /tmp/tmp.j44LZPowal/cluster1-mirror-image_4.status 2026-02-22T21:57:26.570 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:57:26.570 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:26.570 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_4 2026-02-22T21:57:26.571 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_4.status 2026-02-22T21:57:26.571 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_4 --format xml 2026-02-22T21:57:26.571 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:57:26.600 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:57:26.602 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id=92 2026-02-22T21:57:26.602 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 92 = 92 2026-02-22T21:57:26.602 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:57:26.602 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snap_present cluster1 mirror image_4 snap 2026-02-22T21:57:26.602 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:57:26.602 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:26.602 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_4 2026-02-22T21:57:26.602 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T21:57:26.602 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T21:57:26.602 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T21:57:26.602 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T21:57:27.603 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 info mirror/image_4@snap 2026-02-22T21:57:27.636 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_4': 2026-02-22T21:57:27.636 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 128 MiB in 32 objects 2026-02-22T21:57:27.636 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T21:57:27.636 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 2 2026-02-22T21:57:27.636 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 12273b49e4d0 2026-02-22T21:57:27.636 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.12273b49e4d0 2026-02-22T21:57:27.636 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T21:57:27.636 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock, non-primary 2026-02-22T21:57:27.636 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T21:57:27.636 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T21:57:27.636 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:48:24 2026 2026-02-22T21:57:27.636 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:48:24 2026 2026-02-22T21:57:27.636 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:48:24 2026 2026-02-22T21:57:27.636 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: protected: False 2026-02-22T21:57:27.636 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T21:57:27.636 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T21:57:27.636 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: 23c3bee2-bece-4830-b190-6b5f39e55cb1 2026-02-22T21:57:27.636 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: false 2026-02-22T21:57:27.637 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:57:27.638 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ compare_image_snaps mirror image_4 snap 2026-02-22T21:57:27.638 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:27.638 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_4 2026-02-22T21:57:27.638 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T21:57:27.638 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local ret=0 2026-02-22T21:57:27.638 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local rmt_export=/tmp/tmp.j44LZPowal/cluster2-mirror-image_4.export 2026-02-22T21:57:27.638 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local loc_export=/tmp/tmp.j44LZPowal/cluster1-mirror-image_4.export 2026-02-22T21:57:27.638 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-image_4.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_4.export 2026-02-22T21:57:27.638 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror export image_4@snap /tmp/tmp.j44LZPowal/cluster2-mirror-image_4.export 2026-02-22T21:57:27.745 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T21:57:27.753 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 -p mirror export image_4@snap /tmp/tmp.j44LZPowal/cluster1-mirror-image_4.export 2026-02-22T21:57:27.879 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T21:57:27.885 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ cmp /tmp/tmp.j44LZPowal/cluster2-mirror-image_4.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_4.export 2026-02-22T21:57:27.934 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-image_4.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_4.export 2026-02-22T21:57:27.968 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:57:27.968 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T21:57:27.968 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_5 2026-02-22T21:57:27.968 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_snap cluster2 mirror image_5 snap 2026-02-22T21:57:27.968 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:57:27.968 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:27.968 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_5 2026-02-22T21:57:27.968 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T21:57:27.968 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror snap create image_5@snap --debug-rbd=20 --debug-journaler=20 2026-02-22T21:57:29.160 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_started cluster1 mirror image_5 2026-02-22T21:57:29.160 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:57:29.160 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:29.160 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_5 2026-02-22T21:57:29.161 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_state cluster1 mirror image_5 started 2026-02-22T21:57:29.161 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:57:29.161 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:29.161 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_5 2026-02-22T21:57:29.161 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=started 2026-02-22T21:57:29.161 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T21:57:29.162 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:57:29.162 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T21:57:30.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_replay_state cluster1 mirror image_5 started 2026-02-22T21:57:30.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:57:30.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:30.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_5 2026-02-22T21:57:30.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=started 2026-02-22T21:57:30.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_result 2026-02-22T21:57:30.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=stopped 2026-02-22T21:57:30.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_5 2026-02-22T21:57:30.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ grep -i state 2026-02-22T21:57:30.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_instance=cluster1 2026-02-22T21:57:30.169 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:57:30.169 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T21:57:30.169 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance=cluster1 2026-02-22T21:57:30.169 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local loop_instance 2026-02-22T21:57:30.169 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:57:30.169 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sleep 0 2026-02-22T21:57:30.170 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:57:30.170 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ seq 0 3 2026-02-22T21:57:30.170 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:57:30.170 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_5 2026-02-22T21:57:30.170 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:0 2026-02-22T21:57:30.170 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:57:30.170 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:57:30.171 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:57:30.171 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:0 2026-02-22T21:57:30.171 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:57:30.171 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:57:30.171 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:57:30.171 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=0 2026-02-22T21:57:30.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:57:30.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:57:30.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:57:30.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=0 2026-02-22T21:57:30.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T21:57:30.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:57:30.173 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:57:30.173 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:57:30.173 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:57:30.173 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:57:30.173 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:0 2026-02-22T21:57:30.173 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:57:30.173 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:57:30.174 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:57:30.174 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T21:57:30.174 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:57:30.174 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:57:30.174 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:57:30.174 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=0 2026-02-22T21:57:30.175 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=0 2026-02-22T21:57:30.175 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:57:30.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:57:30.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:57:30.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:57:30.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_5 2026-02-22T21:57:30.222 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_5'] 2026-02-22T21:57:30.223 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:57:30.223 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T21:57:30.225 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:57:30.225 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_5 2026-02-22T21:57:30.225 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:1 2026-02-22T21:57:30.225 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:57:30.225 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:57:30.225 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T21:57:30.225 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:1 2026-02-22T21:57:30.225 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:57:30.225 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:57:30.225 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:57:30.225 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=1 2026-02-22T21:57:30.226 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T21:57:30.226 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:57:30.226 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:57:30.226 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=1 2026-02-22T21:57:30.226 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T21:57:30.226 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T21:57:30.226 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:1 2026-02-22T21:57:30.226 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:57:30.226 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:57:30.226 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T21:57:30.226 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:1 2026-02-22T21:57:30.226 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:57:30.226 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:57:30.226 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:57:30.226 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T21:57:30.226 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T21:57:30.226 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:57:30.226 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:57:30.226 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=1 2026-02-22T21:57:30.226 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=1 2026-02-22T21:57:30.226 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T21:57:30.236 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:57:30.237 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:57:30.237 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:57:30.237 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_5 2026-02-22T21:57:30.282 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:no valid command found; 1 closest matches: 2026-02-22T21:57:30.282 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status mirror/image_50 2026-02-22T21:57:30.282 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T21:57:30.283 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:57:30.283 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_5 2026-02-22T21:57:30.283 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:2 2026-02-22T21:57:30.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:57:30.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:57:30.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T21:57:30.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:2 2026-02-22T21:57:30.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:57:30.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:57:30.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:57:30.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=2 2026-02-22T21:57:30.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T21:57:30.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:57:30.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:57:30.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=2 2026-02-22T21:57:30.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T21:57:30.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T21:57:30.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:2 2026-02-22T21:57:30.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:57:30.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:57:30.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T21:57:30.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:2 2026-02-22T21:57:30.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:57:30.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:57:30.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:57:30.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T21:57:30.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T21:57:30.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:57:30.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:57:30.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=2 2026-02-22T21:57:30.285 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=2 2026-02-22T21:57:30.285 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T21:57:30.296 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:57:30.296 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:57:30.296 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:57:30.296 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_5 2026-02-22T21:57:30.348 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ return 0 2026-02-22T21:57:30.349 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:57:30.349 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:57:30.349 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep -i Replaying 2026-02-22T21:57:30.349 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: "state": "Replaying" 2026-02-22T21:57:30.351 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_state=started 2026-02-22T21:57:30.351 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test started = started 2026-02-22T21:57:30.351 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:57:30.351 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_5 2026-02-22T21:57:30.351 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T21:57:30.351 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:57:30.351 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:30.352 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_5 2026-02-22T21:57:30.352 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = journal ']' 2026-02-22T21:57:30.352 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T21:57:30.352 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ mirror_image_snapshot cluster2 mirror image_5 2026-02-22T21:57:30.352 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:57:30.352 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:30.352 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_5 2026-02-22T21:57:30.352 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_5 2026-02-22T21:57:31.165 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Snapshot ID: 94 2026-02-22T21:57:31.171 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_5 2026-02-22T21:57:31.171 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T21:57:31.171 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:57:31.171 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:31.171 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_5 2026-02-22T21:57:31.171 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster2-mirror-image_5.status 2026-02-22T21:57:31.171 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster2-mirror-image_5.status 2026-02-22T21:57:31.171 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T21:57:31.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_log=/tmp/tmp.j44LZPowal/cluster2-mirror-image_5.status 2026-02-22T21:57:31.173 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster1-mirror-image_5.status 2026-02-22T21:57:31.173 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster1-mirror-image_5.status 2026-02-22T21:57:31.174 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T21:57:31.175 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_status_log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_5.status 2026-02-22T21:57:31.175 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_5 /tmp/tmp.j44LZPowal/cluster2-mirror-image_5.status 2026-02-22T21:57:31.175 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:57:31.175 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:31.175 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_5 2026-02-22T21:57:31.175 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster2-mirror-image_5.status 2026-02-22T21:57:31.175 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_5 --format xml 2026-02-22T21:57:31.175 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:57:31.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T21:57:31.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snapshot_id=94 2026-02-22T21:57:31.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:57:31.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:57:31.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.2 2026-02-22T21:57:31.405 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_5 /tmp/tmp.j44LZPowal/cluster1-mirror-image_5.status 2026-02-22T21:57:31.405 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:57:31.405 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:31.405 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_5 2026-02-22T21:57:31.405 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_5.status 2026-02-22T21:57:31.405 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_5 --format xml 2026-02-22T21:57:31.405 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:57:31.434 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:57:31.435 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:57:31.436 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T21:57:31.437 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T21:57:31.437 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T21:57:31.437 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T21:57:31.437 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 94 = '' 2026-02-22T21:57:31.437 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:57:31.437 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.4 2026-02-22T21:57:31.837 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_5 /tmp/tmp.j44LZPowal/cluster1-mirror-image_5.status 2026-02-22T21:57:31.837 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:57:31.837 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:31.837 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_5 2026-02-22T21:57:31.837 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_5.status 2026-02-22T21:57:31.838 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_5 --format xml 2026-02-22T21:57:31.838 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:57:31.867 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:57:31.868 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:57:31.869 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T21:57:31.869 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T21:57:31.869 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T21:57:31.869 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T21:57:31.870 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 94 = '' 2026-02-22T21:57:31.870 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:57:31.870 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.8 2026-02-22T21:57:32.670 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_5 /tmp/tmp.j44LZPowal/cluster1-mirror-image_5.status 2026-02-22T21:57:32.671 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:57:32.671 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:32.671 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_5 2026-02-22T21:57:32.671 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_5.status 2026-02-22T21:57:32.671 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_5 --format xml 2026-02-22T21:57:32.671 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:57:32.701 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:57:32.703 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id=94 2026-02-22T21:57:32.703 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 94 = 94 2026-02-22T21:57:32.703 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:57:32.703 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snap_present cluster1 mirror image_5 snap 2026-02-22T21:57:32.703 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:57:32.703 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:32.703 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_5 2026-02-22T21:57:32.703 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T21:57:32.703 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T21:57:32.703 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T21:57:32.703 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T21:57:33.704 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 info mirror/image_5@snap 2026-02-22T21:57:33.735 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_5': 2026-02-22T21:57:33.735 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 128 MiB in 32 objects 2026-02-22T21:57:33.735 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T21:57:33.735 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 2 2026-02-22T21:57:33.735 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 12277a5f4bd8 2026-02-22T21:57:33.736 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.12277a5f4bd8 2026-02-22T21:57:33.736 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T21:57:33.736 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock, non-primary 2026-02-22T21:57:33.736 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T21:57:33.736 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T21:57:33.736 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:48:59 2026 2026-02-22T21:57:33.736 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:48:59 2026 2026-02-22T21:57:33.736 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:48:59 2026 2026-02-22T21:57:33.736 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: protected: False 2026-02-22T21:57:33.736 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T21:57:33.736 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T21:57:33.737 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: d7359255-a2a8-4833-b9d4-992b53784bf0 2026-02-22T21:57:33.737 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: false 2026-02-22T21:57:33.738 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:57:33.738 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ compare_image_snaps mirror image_5 snap 2026-02-22T21:57:33.738 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:33.738 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_5 2026-02-22T21:57:33.738 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T21:57:33.738 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local ret=0 2026-02-22T21:57:33.738 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local rmt_export=/tmp/tmp.j44LZPowal/cluster2-mirror-image_5.export 2026-02-22T21:57:33.739 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local loc_export=/tmp/tmp.j44LZPowal/cluster1-mirror-image_5.export 2026-02-22T21:57:33.739 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-image_5.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_5.export 2026-02-22T21:57:33.739 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror export image_5@snap /tmp/tmp.j44LZPowal/cluster2-mirror-image_5.export 2026-02-22T21:57:33.847 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T21:57:33.854 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 -p mirror export image_5@snap /tmp/tmp.j44LZPowal/cluster1-mirror-image_5.export 2026-02-22T21:57:33.974 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T21:57:33.982 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ cmp /tmp/tmp.j44LZPowal/cluster2-mirror-image_5.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_5.export 2026-02-22T21:57:34.033 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-image_5.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_5.export 2026-02-22T21:57:34.067 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:57:34.067 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T21:57:34.067 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_6 2026-02-22T21:57:34.067 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_snap cluster2 mirror image_6 snap 2026-02-22T21:57:34.067 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:57:34.068 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:34.068 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_6 2026-02-22T21:57:34.068 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T21:57:34.068 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror snap create image_6@snap --debug-rbd=20 --debug-journaler=20 2026-02-22T21:57:35.169 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_started cluster1 mirror image_6 2026-02-22T21:57:35.170 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:57:35.170 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:35.170 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_6 2026-02-22T21:57:35.170 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_state cluster1 mirror image_6 started 2026-02-22T21:57:35.170 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:57:35.170 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:35.170 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_6 2026-02-22T21:57:35.170 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=started 2026-02-22T21:57:35.170 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T21:57:35.170 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:57:35.171 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T21:57:36.171 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_replay_state cluster1 mirror image_6 started 2026-02-22T21:57:36.171 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:57:36.171 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:36.171 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_6 2026-02-22T21:57:36.171 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=started 2026-02-22T21:57:36.171 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_result 2026-02-22T21:57:36.171 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=stopped 2026-02-22T21:57:36.171 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_6 2026-02-22T21:57:36.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_instance=cluster1 2026-02-22T21:57:36.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:57:36.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T21:57:36.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance=cluster1 2026-02-22T21:57:36.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local loop_instance 2026-02-22T21:57:36.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:57:36.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sleep 0 2026-02-22T21:57:36.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ grep -i state 2026-02-22T21:57:36.175 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:57:36.175 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ seq 0 3 2026-02-22T21:57:36.175 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:57:36.175 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_6 2026-02-22T21:57:36.175 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:0 2026-02-22T21:57:36.175 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:57:36.175 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:57:36.175 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:57:36.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:0 2026-02-22T21:57:36.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:57:36.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:57:36.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:57:36.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=0 2026-02-22T21:57:36.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:57:36.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:57:36.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:57:36.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=0 2026-02-22T21:57:36.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T21:57:36.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:57:36.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:57:36.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:57:36.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:57:36.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:57:36.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:0 2026-02-22T21:57:36.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:57:36.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:57:36.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:57:36.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T21:57:36.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:57:36.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:57:36.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:57:36.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=0 2026-02-22T21:57:36.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=0 2026-02-22T21:57:36.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:57:36.185 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:57:36.185 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:57:36.185 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:57:36.185 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_6 2026-02-22T21:57:36.230 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_6'] 2026-02-22T21:57:36.231 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:57:36.231 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T21:57:36.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:57:36.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_6 2026-02-22T21:57:36.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:1 2026-02-22T21:57:36.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:57:36.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:57:36.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T21:57:36.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:1 2026-02-22T21:57:36.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:57:36.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:57:36.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:57:36.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=1 2026-02-22T21:57:36.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T21:57:36.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:57:36.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:57:36.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=1 2026-02-22T21:57:36.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T21:57:36.233 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T21:57:36.233 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:1 2026-02-22T21:57:36.233 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:57:36.233 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:57:36.233 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T21:57:36.233 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:1 2026-02-22T21:57:36.233 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:57:36.233 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:57:36.233 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:57:36.233 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T21:57:36.233 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T21:57:36.233 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:57:36.233 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:57:36.233 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=1 2026-02-22T21:57:36.233 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=1 2026-02-22T21:57:36.233 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T21:57:36.243 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:57:36.243 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:57:36.243 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:57:36.244 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_6 2026-02-22T21:57:36.294 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ return 0 2026-02-22T21:57:36.295 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:57:36.295 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:57:36.295 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep -i Replaying 2026-02-22T21:57:36.295 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: "state": "Replaying" 2026-02-22T21:57:36.297 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_state=started 2026-02-22T21:57:36.297 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test started = started 2026-02-22T21:57:36.297 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:57:36.297 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_6 2026-02-22T21:57:36.297 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T21:57:36.297 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:57:36.297 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:36.297 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_6 2026-02-22T21:57:36.297 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = journal ']' 2026-02-22T21:57:36.297 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T21:57:36.297 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ mirror_image_snapshot cluster2 mirror image_6 2026-02-22T21:57:36.298 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:57:36.298 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:36.298 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_6 2026-02-22T21:57:36.298 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_6 2026-02-22T21:57:37.171 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Snapshot ID: 96 2026-02-22T21:57:37.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_6 2026-02-22T21:57:37.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T21:57:37.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:57:37.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:37.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_6 2026-02-22T21:57:37.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster2-mirror-image_6.status 2026-02-22T21:57:37.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster2-mirror-image_6.status 2026-02-22T21:57:37.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T21:57:37.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_log=/tmp/tmp.j44LZPowal/cluster2-mirror-image_6.status 2026-02-22T21:57:37.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster1-mirror-image_6.status 2026-02-22T21:57:37.180 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster1-mirror-image_6.status 2026-02-22T21:57:37.180 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T21:57:37.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_status_log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_6.status 2026-02-22T21:57:37.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_6 /tmp/tmp.j44LZPowal/cluster2-mirror-image_6.status 2026-02-22T21:57:37.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:57:37.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:37.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_6 2026-02-22T21:57:37.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster2-mirror-image_6.status 2026-02-22T21:57:37.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_6 --format xml 2026-02-22T21:57:37.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:57:37.209 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T21:57:37.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snapshot_id=96 2026-02-22T21:57:37.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:57:37.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:57:37.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.2 2026-02-22T21:57:37.412 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_6 /tmp/tmp.j44LZPowal/cluster1-mirror-image_6.status 2026-02-22T21:57:37.413 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:57:37.413 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:37.413 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_6 2026-02-22T21:57:37.413 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_6.status 2026-02-22T21:57:37.413 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_6 --format xml 2026-02-22T21:57:37.413 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:57:37.441 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:57:37.441 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:57:37.442 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T21:57:37.442 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T21:57:37.443 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T21:57:37.443 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T21:57:37.444 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 96 = '' 2026-02-22T21:57:37.444 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:57:37.444 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.4 2026-02-22T21:57:37.844 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_6 /tmp/tmp.j44LZPowal/cluster1-mirror-image_6.status 2026-02-22T21:57:37.845 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:57:37.845 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:37.845 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_6 2026-02-22T21:57:37.845 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_6.status 2026-02-22T21:57:37.845 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_6 --format xml 2026-02-22T21:57:37.845 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:57:37.875 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:57:37.876 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:57:37.877 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T21:57:37.877 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T21:57:37.877 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T21:57:37.878 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T21:57:37.878 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 96 = '' 2026-02-22T21:57:37.878 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:57:37.878 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.8 2026-02-22T21:57:38.679 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_6 /tmp/tmp.j44LZPowal/cluster1-mirror-image_6.status 2026-02-22T21:57:38.679 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:57:38.679 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:38.679 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_6 2026-02-22T21:57:38.679 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_6.status 2026-02-22T21:57:38.679 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_6 --format xml 2026-02-22T21:57:38.680 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:57:38.711 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:57:38.713 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id=96 2026-02-22T21:57:38.714 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 96 = 96 2026-02-22T21:57:38.714 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:57:38.714 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snap_present cluster1 mirror image_6 snap 2026-02-22T21:57:38.714 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:57:38.714 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:38.714 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_6 2026-02-22T21:57:38.714 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T21:57:38.715 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T21:57:38.715 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T21:57:38.715 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T21:57:39.713 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 info mirror/image_6@snap 2026-02-22T21:57:39.743 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_6': 2026-02-22T21:57:39.743 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 128 MiB in 32 objects 2026-02-22T21:57:39.743 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T21:57:39.743 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 2 2026-02-22T21:57:39.743 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 12485bf1339a 2026-02-22T21:57:39.743 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.12485bf1339a 2026-02-22T21:57:39.743 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T21:57:39.743 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock, non-primary 2026-02-22T21:57:39.744 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T21:57:39.744 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T21:57:39.744 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:48:46 2026 2026-02-22T21:57:39.744 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:48:46 2026 2026-02-22T21:57:39.744 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:48:46 2026 2026-02-22T21:57:39.744 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: protected: False 2026-02-22T21:57:39.744 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T21:57:39.744 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T21:57:39.745 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: 763b8c73-d0f5-43b5-9453-e1f95dcf6809 2026-02-22T21:57:39.745 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: false 2026-02-22T21:57:39.746 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:57:39.746 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ compare_image_snaps mirror image_6 snap 2026-02-22T21:57:39.746 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:39.746 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_6 2026-02-22T21:57:39.746 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T21:57:39.746 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local ret=0 2026-02-22T21:57:39.746 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local rmt_export=/tmp/tmp.j44LZPowal/cluster2-mirror-image_6.export 2026-02-22T21:57:39.747 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local loc_export=/tmp/tmp.j44LZPowal/cluster1-mirror-image_6.export 2026-02-22T21:57:39.747 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-image_6.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_6.export 2026-02-22T21:57:39.747 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror export image_6@snap /tmp/tmp.j44LZPowal/cluster2-mirror-image_6.export 2026-02-22T21:57:39.851 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T21:57:39.859 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 -p mirror export image_6@snap /tmp/tmp.j44LZPowal/cluster1-mirror-image_6.export 2026-02-22T21:57:39.984 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T21:57:39.991 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ cmp /tmp/tmp.j44LZPowal/cluster2-mirror-image_6.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_6.export 2026-02-22T21:57:40.040 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-image_6.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_6.export 2026-02-22T21:57:40.074 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:57:40.074 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T21:57:40.074 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_7 2026-02-22T21:57:40.074 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_snap cluster2 mirror image_7 snap 2026-02-22T21:57:40.075 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:57:40.075 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:40.075 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_7 2026-02-22T21:57:40.075 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T21:57:40.075 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror snap create image_7@snap --debug-rbd=20 --debug-journaler=20 2026-02-22T21:57:41.174 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_started cluster1 mirror image_7 2026-02-22T21:57:41.174 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:57:41.174 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:41.174 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_7 2026-02-22T21:57:41.174 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_state cluster1 mirror image_7 started 2026-02-22T21:57:41.174 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:57:41.174 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:41.175 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_7 2026-02-22T21:57:41.175 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=started 2026-02-22T21:57:41.175 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T21:57:41.175 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:57:41.175 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T21:57:42.175 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_replay_state cluster1 mirror image_7 started 2026-02-22T21:57:42.175 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:57:42.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:42.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_7 2026-02-22T21:57:42.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=started 2026-02-22T21:57:42.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_result 2026-02-22T21:57:42.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=stopped 2026-02-22T21:57:42.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_7 2026-02-22T21:57:42.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_instance=cluster1 2026-02-22T21:57:42.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:57:42.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T21:57:42.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance=cluster1 2026-02-22T21:57:42.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local loop_instance 2026-02-22T21:57:42.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:57:42.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sleep 0 2026-02-22T21:57:42.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ grep -i state 2026-02-22T21:57:42.180 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:57:42.180 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ seq 0 3 2026-02-22T21:57:42.180 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:57:42.180 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_7 2026-02-22T21:57:42.180 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:0 2026-02-22T21:57:42.180 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:57:42.180 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:57:42.180 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:57:42.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:0 2026-02-22T21:57:42.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:57:42.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:57:42.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:57:42.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=0 2026-02-22T21:57:42.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:57:42.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:57:42.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:57:42.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=0 2026-02-22T21:57:42.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T21:57:42.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:57:42.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:57:42.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:57:42.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:57:42.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:57:42.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:0 2026-02-22T21:57:42.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:57:42.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:57:42.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:57:42.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T21:57:42.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:57:42.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:57:42.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:57:42.183 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=0 2026-02-22T21:57:42.183 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=0 2026-02-22T21:57:42.183 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:57:42.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:57:42.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:57:42.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:57:42.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_7 2026-02-22T21:57:42.239 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_7'] 2026-02-22T21:57:42.239 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:57:42.239 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T21:57:42.242 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:57:42.242 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_7 2026-02-22T21:57:42.242 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:1 2026-02-22T21:57:42.242 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:57:42.242 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:57:42.242 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T21:57:42.243 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:1 2026-02-22T21:57:42.243 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:57:42.243 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:57:42.243 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:57:42.243 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=1 2026-02-22T21:57:42.243 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T21:57:42.243 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:57:42.243 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:57:42.243 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=1 2026-02-22T21:57:42.243 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T21:57:42.243 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T21:57:42.243 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:1 2026-02-22T21:57:42.244 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:57:42.244 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:57:42.244 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T21:57:42.244 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:1 2026-02-22T21:57:42.244 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:57:42.244 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:57:42.244 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:57:42.244 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T21:57:42.244 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T21:57:42.244 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:57:42.244 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:57:42.244 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=1 2026-02-22T21:57:42.244 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=1 2026-02-22T21:57:42.245 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T21:57:42.252 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:57:42.252 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:57:42.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:57:42.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_7 2026-02-22T21:57:42.297 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_7'] 2026-02-22T21:57:42.297 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:57:42.297 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T21:57:42.298 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:57:42.298 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_7 2026-02-22T21:57:42.298 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:2 2026-02-22T21:57:42.298 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:57:42.298 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:57:42.298 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T21:57:42.298 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:2 2026-02-22T21:57:42.298 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:57:42.298 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:57:42.298 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:57:42.298 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=2 2026-02-22T21:57:42.298 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T21:57:42.298 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:57:42.298 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:57:42.298 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=2 2026-02-22T21:57:42.298 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T21:57:42.299 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T21:57:42.299 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:2 2026-02-22T21:57:42.299 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:57:42.299 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:57:42.299 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T21:57:42.299 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:2 2026-02-22T21:57:42.299 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:57:42.299 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:57:42.299 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:57:42.299 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T21:57:42.299 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T21:57:42.299 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:57:42.299 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:57:42.299 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=2 2026-02-22T21:57:42.299 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=2 2026-02-22T21:57:42.299 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T21:57:42.310 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:57:42.310 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:57:42.310 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:57:42.311 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_7 2026-02-22T21:57:42.358 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ return 0 2026-02-22T21:57:42.359 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:57:42.359 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:57:42.359 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep -i Replaying 2026-02-22T21:57:42.360 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: "state": "Replaying" 2026-02-22T21:57:42.361 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_state=started 2026-02-22T21:57:42.361 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test started = started 2026-02-22T21:57:42.361 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:57:42.362 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_7 2026-02-22T21:57:42.362 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T21:57:42.362 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:57:42.362 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:42.362 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_7 2026-02-22T21:57:42.362 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = journal ']' 2026-02-22T21:57:42.362 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T21:57:42.362 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ mirror_image_snapshot cluster2 mirror image_7 2026-02-22T21:57:42.362 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:57:42.362 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:42.362 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_7 2026-02-22T21:57:42.363 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_7 2026-02-22T21:57:43.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Snapshot ID: 98 2026-02-22T21:57:43.183 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_7 2026-02-22T21:57:43.183 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T21:57:43.183 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:57:43.183 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:43.184 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_7 2026-02-22T21:57:43.184 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster2-mirror-image_7.status 2026-02-22T21:57:43.184 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster2-mirror-image_7.status 2026-02-22T21:57:43.184 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T21:57:43.185 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_log=/tmp/tmp.j44LZPowal/cluster2-mirror-image_7.status 2026-02-22T21:57:43.185 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster1-mirror-image_7.status 2026-02-22T21:57:43.185 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster1-mirror-image_7.status 2026-02-22T21:57:43.185 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T21:57:43.187 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_status_log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_7.status 2026-02-22T21:57:43.187 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_7 /tmp/tmp.j44LZPowal/cluster2-mirror-image_7.status 2026-02-22T21:57:43.187 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:57:43.187 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:43.187 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_7 2026-02-22T21:57:43.187 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster2-mirror-image_7.status 2026-02-22T21:57:43.187 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_7 --format xml 2026-02-22T21:57:43.187 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:57:43.214 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T21:57:43.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snapshot_id=98 2026-02-22T21:57:43.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:57:43.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:57:43.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.2 2026-02-22T21:57:43.418 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_7 /tmp/tmp.j44LZPowal/cluster1-mirror-image_7.status 2026-02-22T21:57:43.418 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:57:43.418 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:43.418 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_7 2026-02-22T21:57:43.418 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_7.status 2026-02-22T21:57:43.418 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_7 --format xml 2026-02-22T21:57:43.418 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:57:43.447 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:57:43.447 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:57:43.448 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T21:57:43.448 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T21:57:43.448 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T21:57:43.449 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T21:57:43.449 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 98 = '' 2026-02-22T21:57:43.449 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:57:43.450 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.4 2026-02-22T21:57:43.850 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_7 /tmp/tmp.j44LZPowal/cluster1-mirror-image_7.status 2026-02-22T21:57:43.850 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:57:43.850 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:43.851 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_7 2026-02-22T21:57:43.851 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_7.status 2026-02-22T21:57:43.851 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_7 --format xml 2026-02-22T21:57:43.851 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:57:43.880 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:57:43.880 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:57:43.881 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T21:57:43.881 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T21:57:43.881 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T21:57:43.882 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T21:57:43.882 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 98 = '' 2026-02-22T21:57:43.883 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:57:43.883 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.8 2026-02-22T21:57:44.683 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_7 /tmp/tmp.j44LZPowal/cluster1-mirror-image_7.status 2026-02-22T21:57:44.683 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:57:44.683 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:44.683 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_7 2026-02-22T21:57:44.683 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_7.status 2026-02-22T21:57:44.683 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_7 --format xml 2026-02-22T21:57:44.683 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:57:44.713 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:57:44.715 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id=98 2026-02-22T21:57:44.716 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 98 = 98 2026-02-22T21:57:44.716 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:57:44.716 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snap_present cluster1 mirror image_7 snap 2026-02-22T21:57:44.716 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:57:44.716 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:44.716 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_7 2026-02-22T21:57:44.716 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T21:57:44.716 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T21:57:44.716 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T21:57:44.716 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T21:57:45.716 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 info mirror/image_7@snap 2026-02-22T21:57:45.744 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_7': 2026-02-22T21:57:45.744 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 128 MiB in 32 objects 2026-02-22T21:57:45.744 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T21:57:45.744 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 2 2026-02-22T21:57:45.744 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 119accea6216 2026-02-22T21:57:45.744 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.119accea6216 2026-02-22T21:57:45.744 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T21:57:45.744 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock, non-primary 2026-02-22T21:57:45.745 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T21:57:45.745 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T21:57:45.745 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:48:55 2026 2026-02-22T21:57:45.745 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:48:55 2026 2026-02-22T21:57:45.745 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:48:55 2026 2026-02-22T21:57:45.745 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: protected: False 2026-02-22T21:57:45.745 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T21:57:45.745 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T21:57:45.745 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: a383f702-7739-4131-97b5-30656d4ff503 2026-02-22T21:57:45.745 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: false 2026-02-22T21:57:45.747 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:57:45.747 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ compare_image_snaps mirror image_7 snap 2026-02-22T21:57:45.747 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:45.747 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_7 2026-02-22T21:57:45.748 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T21:57:45.748 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local ret=0 2026-02-22T21:57:45.748 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local rmt_export=/tmp/tmp.j44LZPowal/cluster2-mirror-image_7.export 2026-02-22T21:57:45.748 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local loc_export=/tmp/tmp.j44LZPowal/cluster1-mirror-image_7.export 2026-02-22T21:57:45.748 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-image_7.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_7.export 2026-02-22T21:57:45.748 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror export image_7@snap /tmp/tmp.j44LZPowal/cluster2-mirror-image_7.export 2026-02-22T21:57:45.854 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T21:57:45.860 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 -p mirror export image_7@snap /tmp/tmp.j44LZPowal/cluster1-mirror-image_7.export 2026-02-22T21:57:45.981 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T21:57:45.990 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ cmp /tmp/tmp.j44LZPowal/cluster2-mirror-image_7.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_7.export 2026-02-22T21:57:46.041 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-image_7.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_7.export 2026-02-22T21:57:46.075 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:57:46.075 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T21:57:46.075 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_8 2026-02-22T21:57:46.075 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_snap cluster2 mirror image_8 snap 2026-02-22T21:57:46.075 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:57:46.075 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:46.075 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_8 2026-02-22T21:57:46.075 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T21:57:46.075 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror snap create image_8@snap --debug-rbd=20 --debug-journaler=20 2026-02-22T21:57:47.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_started cluster1 mirror image_8 2026-02-22T21:57:47.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:57:47.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:47.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_8 2026-02-22T21:57:47.183 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_state cluster1 mirror image_8 started 2026-02-22T21:57:47.183 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:57:47.183 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:47.183 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_8 2026-02-22T21:57:47.183 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=started 2026-02-22T21:57:47.183 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T21:57:47.183 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:57:47.183 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T21:57:48.184 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_replay_state cluster1 mirror image_8 started 2026-02-22T21:57:48.184 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:57:48.184 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:48.184 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_8 2026-02-22T21:57:48.184 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=started 2026-02-22T21:57:48.184 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_result 2026-02-22T21:57:48.184 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=stopped 2026-02-22T21:57:48.184 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_8 2026-02-22T21:57:48.185 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_instance=cluster1 2026-02-22T21:57:48.185 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:57:48.185 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ grep -i state 2026-02-22T21:57:48.185 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T21:57:48.185 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance=cluster1 2026-02-22T21:57:48.185 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local loop_instance 2026-02-22T21:57:48.185 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:57:48.185 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sleep 0 2026-02-22T21:57:48.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:57:48.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ seq 0 3 2026-02-22T21:57:48.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:57:48.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_8 2026-02-22T21:57:48.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:0 2026-02-22T21:57:48.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:57:48.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:57:48.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:57:48.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:0 2026-02-22T21:57:48.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:57:48.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:57:48.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:57:48.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=0 2026-02-22T21:57:48.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:57:48.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:57:48.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:57:48.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=0 2026-02-22T21:57:48.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T21:57:48.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:57:48.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:57:48.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:57:48.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:57:48.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:57:48.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:0 2026-02-22T21:57:48.195 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:57:48.195 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:57:48.195 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:57:48.195 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T21:57:48.195 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:57:48.196 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:57:48.196 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:57:48.196 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=0 2026-02-22T21:57:48.196 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=0 2026-02-22T21:57:48.196 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:57:48.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:57:48.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:57:48.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:57:48.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_8 2026-02-22T21:57:48.250 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ return 0 2026-02-22T21:57:48.250 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:57:48.250 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:57:48.251 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep -i Replaying 2026-02-22T21:57:48.251 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: "state": "Replaying" 2026-02-22T21:57:48.252 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_state=started 2026-02-22T21:57:48.252 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test started = started 2026-02-22T21:57:48.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:57:48.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_8 2026-02-22T21:57:48.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T21:57:48.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:57:48.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:48.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_8 2026-02-22T21:57:48.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = journal ']' 2026-02-22T21:57:48.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T21:57:48.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ mirror_image_snapshot cluster2 mirror image_8 2026-02-22T21:57:48.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:57:48.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:48.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_8 2026-02-22T21:57:48.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_8 2026-02-22T21:57:49.185 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Snapshot ID: 100 2026-02-22T21:57:49.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_8 2026-02-22T21:57:49.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T21:57:49.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:57:49.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:49.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_8 2026-02-22T21:57:49.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster2-mirror-image_8.status 2026-02-22T21:57:49.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster2-mirror-image_8.status 2026-02-22T21:57:49.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T21:57:49.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_log=/tmp/tmp.j44LZPowal/cluster2-mirror-image_8.status 2026-02-22T21:57:49.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster1-mirror-image_8.status 2026-02-22T21:57:49.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster1-mirror-image_8.status 2026-02-22T21:57:49.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T21:57:49.195 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_status_log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_8.status 2026-02-22T21:57:49.195 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_8 /tmp/tmp.j44LZPowal/cluster2-mirror-image_8.status 2026-02-22T21:57:49.195 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:57:49.195 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:49.196 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_8 2026-02-22T21:57:49.196 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster2-mirror-image_8.status 2026-02-22T21:57:49.196 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_8 --format xml 2026-02-22T21:57:49.196 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:57:49.223 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T21:57:49.225 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snapshot_id=100 2026-02-22T21:57:49.225 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:57:49.225 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:57:49.225 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.2 2026-02-22T21:57:49.426 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_8 /tmp/tmp.j44LZPowal/cluster1-mirror-image_8.status 2026-02-22T21:57:49.426 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:57:49.426 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:49.426 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_8 2026-02-22T21:57:49.427 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_8.status 2026-02-22T21:57:49.427 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_8 --format xml 2026-02-22T21:57:49.427 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:57:49.458 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:57:49.458 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:57:49.458 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T21:57:49.459 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T21:57:49.459 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T21:57:49.459 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T21:57:49.460 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 100 = '' 2026-02-22T21:57:49.460 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:57:49.460 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.4 2026-02-22T21:57:49.860 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_8 /tmp/tmp.j44LZPowal/cluster1-mirror-image_8.status 2026-02-22T21:57:49.861 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:57:49.861 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:49.861 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_8 2026-02-22T21:57:49.861 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_8.status 2026-02-22T21:57:49.861 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_8 --format xml 2026-02-22T21:57:49.861 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:57:49.889 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:57:49.890 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:57:49.890 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T21:57:49.891 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T21:57:49.891 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T21:57:49.891 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T21:57:49.891 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 100 = '' 2026-02-22T21:57:49.892 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:57:49.892 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.8 2026-02-22T21:57:50.692 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_8 /tmp/tmp.j44LZPowal/cluster1-mirror-image_8.status 2026-02-22T21:57:50.693 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:57:50.693 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:50.693 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_8 2026-02-22T21:57:50.693 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_8.status 2026-02-22T21:57:50.693 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_8 --format xml 2026-02-22T21:57:50.693 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:57:50.724 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:57:50.726 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id=100 2026-02-22T21:57:50.726 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 100 = 100 2026-02-22T21:57:50.727 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:57:50.727 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snap_present cluster1 mirror image_8 snap 2026-02-22T21:57:50.727 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:57:50.727 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:50.727 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_8 2026-02-22T21:57:50.727 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T21:57:50.727 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T21:57:50.727 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T21:57:50.727 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T21:57:51.727 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 info mirror/image_8@snap 2026-02-22T21:57:51.755 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_8': 2026-02-22T21:57:51.755 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 128 MiB in 32 objects 2026-02-22T21:57:51.755 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T21:57:51.755 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 2 2026-02-22T21:57:51.755 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 1248f92cd070 2026-02-22T21:57:51.756 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.1248f92cd070 2026-02-22T21:57:51.756 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T21:57:51.756 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock, non-primary 2026-02-22T21:57:51.756 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T21:57:51.756 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T21:57:51.756 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:49:02 2026 2026-02-22T21:57:51.756 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:49:02 2026 2026-02-22T21:57:51.756 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:49:02 2026 2026-02-22T21:57:51.756 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: protected: False 2026-02-22T21:57:51.756 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T21:57:51.756 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T21:57:51.757 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: 6ec8d83c-acbe-4670-87cf-a72aa84767b5 2026-02-22T21:57:51.757 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: false 2026-02-22T21:57:51.758 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:57:51.758 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ compare_image_snaps mirror image_8 snap 2026-02-22T21:57:51.758 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:51.758 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_8 2026-02-22T21:57:51.758 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T21:57:51.758 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local ret=0 2026-02-22T21:57:51.758 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local rmt_export=/tmp/tmp.j44LZPowal/cluster2-mirror-image_8.export 2026-02-22T21:57:51.758 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local loc_export=/tmp/tmp.j44LZPowal/cluster1-mirror-image_8.export 2026-02-22T21:57:51.759 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-image_8.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_8.export 2026-02-22T21:57:51.759 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror export image_8@snap /tmp/tmp.j44LZPowal/cluster2-mirror-image_8.export 2026-02-22T21:57:51.858 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T21:57:51.865 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 -p mirror export image_8@snap /tmp/tmp.j44LZPowal/cluster1-mirror-image_8.export 2026-02-22T21:57:51.994 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T21:57:52.002 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ cmp /tmp/tmp.j44LZPowal/cluster2-mirror-image_8.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_8.export 2026-02-22T21:57:52.050 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-image_8.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_8.export 2026-02-22T21:57:52.084 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:57:52.084 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T21:57:52.084 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_9 2026-02-22T21:57:52.084 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_snap cluster2 mirror image_9 snap 2026-02-22T21:57:52.084 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:57:52.085 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:52.085 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_9 2026-02-22T21:57:52.085 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T21:57:52.085 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror snap create image_9@snap --debug-rbd=20 --debug-journaler=20 2026-02-22T21:57:53.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_started cluster1 mirror image_9 2026-02-22T21:57:53.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:57:53.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:53.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_9 2026-02-22T21:57:53.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_state cluster1 mirror image_9 started 2026-02-22T21:57:53.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:57:53.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:53.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_9 2026-02-22T21:57:53.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=started 2026-02-22T21:57:53.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T21:57:53.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:57:53.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T21:57:54.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_replay_state cluster1 mirror image_9 started 2026-02-22T21:57:54.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:57:54.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:54.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_9 2026-02-22T21:57:54.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=started 2026-02-22T21:57:54.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_result 2026-02-22T21:57:54.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=stopped 2026-02-22T21:57:54.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_9 2026-02-22T21:57:54.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_instance=cluster1 2026-02-22T21:57:54.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:57:54.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T21:57:54.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance=cluster1 2026-02-22T21:57:54.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local loop_instance 2026-02-22T21:57:54.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:57:54.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sleep 0 2026-02-22T21:57:54.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ grep -i state 2026-02-22T21:57:54.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:57:54.195 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ seq 0 3 2026-02-22T21:57:54.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:57:54.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_9 2026-02-22T21:57:54.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:0 2026-02-22T21:57:54.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:57:54.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:57:54.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:57:54.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:0 2026-02-22T21:57:54.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:57:54.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:57:54.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:57:54.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=0 2026-02-22T21:57:54.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:57:54.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:57:54.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:57:54.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=0 2026-02-22T21:57:54.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T21:57:54.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:57:54.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:57:54.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:57:54.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:57:54.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:57:54.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:0 2026-02-22T21:57:54.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:57:54.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:57:54.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:57:54.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T21:57:54.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:57:54.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:57:54.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:57:54.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=0 2026-02-22T21:57:54.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=0 2026-02-22T21:57:54.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:57:54.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:57:54.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:57:54.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:57:54.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_9 2026-02-22T21:57:54.254 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_9'] 2026-02-22T21:57:54.254 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:57:54.254 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T21:57:54.256 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:57:54.256 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_9 2026-02-22T21:57:54.256 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:1 2026-02-22T21:57:54.256 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:57:54.256 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:57:54.256 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T21:57:54.256 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:1 2026-02-22T21:57:54.256 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:57:54.256 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:57:54.256 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:57:54.256 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=1 2026-02-22T21:57:54.256 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T21:57:54.256 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:57:54.256 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:57:54.256 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=1 2026-02-22T21:57:54.256 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T21:57:54.256 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T21:57:54.257 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:1 2026-02-22T21:57:54.257 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:57:54.257 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:57:54.257 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T21:57:54.257 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:1 2026-02-22T21:57:54.257 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:57:54.257 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:57:54.257 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:57:54.257 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T21:57:54.257 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T21:57:54.257 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:57:54.257 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:57:54.257 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=1 2026-02-22T21:57:54.257 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=1 2026-02-22T21:57:54.257 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T21:57:54.268 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:57:54.269 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:57:54.269 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:57:54.269 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_9 2026-02-22T21:57:54.313 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_9'] 2026-02-22T21:57:54.313 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:57:54.313 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T21:57:54.316 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:57:54.316 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_9 2026-02-22T21:57:54.317 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:2 2026-02-22T21:57:54.317 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:57:54.317 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:57:54.317 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T21:57:54.317 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:2 2026-02-22T21:57:54.317 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:57:54.317 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:57:54.317 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:57:54.317 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=2 2026-02-22T21:57:54.317 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T21:57:54.317 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:57:54.318 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:57:54.318 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=2 2026-02-22T21:57:54.318 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T21:57:54.318 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T21:57:54.318 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:2 2026-02-22T21:57:54.318 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:57:54.318 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:57:54.318 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T21:57:54.318 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:2 2026-02-22T21:57:54.318 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:57:54.318 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:57:54.318 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:57:54.318 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T21:57:54.319 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T21:57:54.319 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:57:54.319 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:57:54.319 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=2 2026-02-22T21:57:54.319 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=2 2026-02-22T21:57:54.319 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T21:57:54.327 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:57:54.327 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:57:54.328 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:57:54.328 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_9 2026-02-22T21:57:54.378 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ return 0 2026-02-22T21:57:54.379 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:57:54.379 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:57:54.379 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep -i Replaying 2026-02-22T21:57:54.379 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: "state": "Replaying" 2026-02-22T21:57:54.381 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_state=started 2026-02-22T21:57:54.381 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test started = started 2026-02-22T21:57:54.381 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:57:54.381 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_9 2026-02-22T21:57:54.381 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T21:57:54.381 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:57:54.381 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:54.381 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_9 2026-02-22T21:57:54.381 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = journal ']' 2026-02-22T21:57:54.381 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T21:57:54.382 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ mirror_image_snapshot cluster2 mirror image_9 2026-02-22T21:57:54.382 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:57:54.382 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:54.382 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_9 2026-02-22T21:57:54.382 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_9 2026-02-22T21:57:55.195 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Snapshot ID: 102 2026-02-22T21:57:55.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_9 2026-02-22T21:57:55.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T21:57:55.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:57:55.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:55.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_9 2026-02-22T21:57:55.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster2-mirror-image_9.status 2026-02-22T21:57:55.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster2-mirror-image_9.status 2026-02-22T21:57:55.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T21:57:55.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_log=/tmp/tmp.j44LZPowal/cluster2-mirror-image_9.status 2026-02-22T21:57:55.203 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster1-mirror-image_9.status 2026-02-22T21:57:55.203 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster1-mirror-image_9.status 2026-02-22T21:57:55.203 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T21:57:55.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_status_log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_9.status 2026-02-22T21:57:55.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_9 /tmp/tmp.j44LZPowal/cluster2-mirror-image_9.status 2026-02-22T21:57:55.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:57:55.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:55.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_9 2026-02-22T21:57:55.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster2-mirror-image_9.status 2026-02-22T21:57:55.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_9 --format xml 2026-02-22T21:57:55.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:57:55.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T21:57:55.234 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snapshot_id=102 2026-02-22T21:57:55.234 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:57:55.234 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:57:55.234 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.2 2026-02-22T21:57:55.435 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_9 /tmp/tmp.j44LZPowal/cluster1-mirror-image_9.status 2026-02-22T21:57:55.435 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:57:55.435 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:55.436 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_9 2026-02-22T21:57:55.436 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_9.status 2026-02-22T21:57:55.436 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_9 --format xml 2026-02-22T21:57:55.436 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:57:55.465 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:57:55.465 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:57:55.466 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T21:57:55.466 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T21:57:55.466 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T21:57:55.467 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T21:57:55.467 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 102 = '' 2026-02-22T21:57:55.467 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:57:55.467 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.4 2026-02-22T21:57:55.868 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_9 /tmp/tmp.j44LZPowal/cluster1-mirror-image_9.status 2026-02-22T21:57:55.868 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:57:55.868 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:55.869 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_9 2026-02-22T21:57:55.869 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_9.status 2026-02-22T21:57:55.869 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_9 --format xml 2026-02-22T21:57:55.869 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:57:55.899 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:57:55.900 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:57:55.900 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T21:57:55.900 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T21:57:55.900 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T21:57:55.901 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T21:57:55.901 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 102 = '' 2026-02-22T21:57:55.901 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:57:55.901 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.8 2026-02-22T21:57:56.703 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_9 /tmp/tmp.j44LZPowal/cluster1-mirror-image_9.status 2026-02-22T21:57:56.703 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:57:56.703 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:56.703 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_9 2026-02-22T21:57:56.704 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_9.status 2026-02-22T21:57:56.704 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_9 --format xml 2026-02-22T21:57:56.704 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:57:56.735 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:57:56.737 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id=102 2026-02-22T21:57:56.737 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 102 = 102 2026-02-22T21:57:56.738 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:57:56.738 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snap_present cluster1 mirror image_9 snap 2026-02-22T21:57:56.738 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:57:56.738 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:56.738 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_9 2026-02-22T21:57:56.738 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T21:57:56.738 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T21:57:56.738 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T21:57:56.738 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T21:57:57.738 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 info mirror/image_9@snap 2026-02-22T21:57:57.763 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_9': 2026-02-22T21:57:57.763 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 128 MiB in 32 objects 2026-02-22T21:57:57.763 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T21:57:57.764 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 2 2026-02-22T21:57:57.764 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 119a5aeb017d 2026-02-22T21:57:57.764 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.119a5aeb017d 2026-02-22T21:57:57.764 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T21:57:57.764 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock, non-primary 2026-02-22T21:57:57.764 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T21:57:57.764 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T21:57:57.764 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:49:14 2026 2026-02-22T21:57:57.764 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:49:14 2026 2026-02-22T21:57:57.764 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:49:14 2026 2026-02-22T21:57:57.764 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: protected: False 2026-02-22T21:57:57.764 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T21:57:57.765 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T21:57:57.765 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: cdba1f90-5b2a-43f5-aa74-a2544c65d250 2026-02-22T21:57:57.765 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: false 2026-02-22T21:57:57.766 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:57:57.766 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ compare_image_snaps mirror image_9 snap 2026-02-22T21:57:57.766 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:57.766 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_9 2026-02-22T21:57:57.766 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T21:57:57.766 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local ret=0 2026-02-22T21:57:57.766 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local rmt_export=/tmp/tmp.j44LZPowal/cluster2-mirror-image_9.export 2026-02-22T21:57:57.766 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local loc_export=/tmp/tmp.j44LZPowal/cluster1-mirror-image_9.export 2026-02-22T21:57:57.767 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-image_9.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_9.export 2026-02-22T21:57:57.767 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror export image_9@snap /tmp/tmp.j44LZPowal/cluster2-mirror-image_9.export 2026-02-22T21:57:58.065 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T21:57:58.072 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 -p mirror export image_9@snap /tmp/tmp.j44LZPowal/cluster1-mirror-image_9.export 2026-02-22T21:57:58.380 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T21:57:58.387 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ cmp /tmp/tmp.j44LZPowal/cluster2-mirror-image_9.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_9.export 2026-02-22T21:57:58.436 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-image_9.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_9.export 2026-02-22T21:57:58.471 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:57:58.471 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T21:57:58.471 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_10 2026-02-22T21:57:58.471 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_snap cluster2 mirror image_10 snap 2026-02-22T21:57:58.471 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:57:58.471 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:58.472 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_10 2026-02-22T21:57:58.472 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T21:57:58.472 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror snap create image_10@snap --debug-rbd=20 --debug-journaler=20 2026-02-22T21:57:59.569 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_started cluster1 mirror image_10 2026-02-22T21:57:59.570 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:57:59.570 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:59.570 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_10 2026-02-22T21:57:59.570 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_state cluster1 mirror image_10 started 2026-02-22T21:57:59.570 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:57:59.570 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:57:59.570 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_10 2026-02-22T21:57:59.570 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=started 2026-02-22T21:57:59.570 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T21:57:59.570 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:57:59.570 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T21:58:00.572 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_replay_state cluster1 mirror image_10 started 2026-02-22T21:58:00.572 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:58:00.572 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:58:00.572 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_10 2026-02-22T21:58:00.572 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=started 2026-02-22T21:58:00.573 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_result 2026-02-22T21:58:00.573 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=stopped 2026-02-22T21:58:00.573 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_10 2026-02-22T21:58:00.573 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_instance=cluster1 2026-02-22T21:58:00.573 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:58:00.573 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T21:58:00.573 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ grep -i state 2026-02-22T21:58:00.573 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance=cluster1 2026-02-22T21:58:00.573 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local loop_instance 2026-02-22T21:58:00.573 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:58:00.574 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sleep 0 2026-02-22T21:58:00.576 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:58:00.577 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ seq 0 3 2026-02-22T21:58:00.577 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:58:00.577 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_10 2026-02-22T21:58:00.577 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:0 2026-02-22T21:58:00.577 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:58:00.577 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:58:00.577 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:58:00.577 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:0 2026-02-22T21:58:00.577 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:58:00.577 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:58:00.577 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:58:00.578 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=0 2026-02-22T21:58:00.578 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:58:00.578 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:58:00.578 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:58:00.578 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=0 2026-02-22T21:58:00.578 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T21:58:00.578 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:58:00.578 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:58:00.578 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:58:00.578 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:58:00.578 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:58:00.578 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:0 2026-02-22T21:58:00.579 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:58:00.579 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:58:00.579 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:58:00.579 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T21:58:00.579 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:58:00.579 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:58:00.579 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:58:00.579 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=0 2026-02-22T21:58:00.579 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=0 2026-02-22T21:58:00.579 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:58:00.587 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:58:00.587 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:58:00.587 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:58:00.587 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_10 2026-02-22T21:58:00.632 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_10'] 2026-02-22T21:58:00.632 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:58:00.632 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T21:58:00.633 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:58:00.634 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_10 2026-02-22T21:58:00.634 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:1 2026-02-22T21:58:00.634 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:58:00.634 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:58:00.634 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T21:58:00.634 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:1 2026-02-22T21:58:00.634 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:58:00.634 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:58:00.634 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:58:00.634 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=1 2026-02-22T21:58:00.634 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T21:58:00.634 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:58:00.634 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:58:00.634 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=1 2026-02-22T21:58:00.634 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T21:58:00.634 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T21:58:00.634 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:1 2026-02-22T21:58:00.634 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:58:00.634 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:58:00.634 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T21:58:00.634 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:1 2026-02-22T21:58:00.634 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:58:00.634 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:58:00.634 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:58:00.635 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T21:58:00.635 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T21:58:00.635 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:58:00.635 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:58:00.635 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=1 2026-02-22T21:58:00.635 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=1 2026-02-22T21:58:00.635 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T21:58:00.645 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:58:00.646 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:58:00.646 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:58:00.646 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_10 2026-02-22T21:58:00.690 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_10'] 2026-02-22T21:58:00.690 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:58:00.690 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T21:58:00.692 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:58:00.692 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_10 2026-02-22T21:58:00.692 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:2 2026-02-22T21:58:00.692 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:58:00.692 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:58:00.692 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T21:58:00.692 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:2 2026-02-22T21:58:00.692 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:58:00.692 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:58:00.692 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:58:00.692 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=2 2026-02-22T21:58:00.692 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T21:58:00.693 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:58:00.693 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:58:00.693 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=2 2026-02-22T21:58:00.693 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T21:58:00.693 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T21:58:00.693 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:2 2026-02-22T21:58:00.693 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:58:00.693 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:58:00.693 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T21:58:00.693 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:2 2026-02-22T21:58:00.693 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:58:00.693 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:58:00.693 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:58:00.693 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T21:58:00.693 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T21:58:00.693 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:58:00.693 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:58:00.693 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=2 2026-02-22T21:58:00.694 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=2 2026-02-22T21:58:00.694 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T21:58:00.703 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:58:00.704 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:58:00.704 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:58:00.704 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_10 2026-02-22T21:58:00.749 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_10'] 2026-02-22T21:58:00.749 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:58:00.749 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T21:58:00.752 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:58:00.753 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_10 2026-02-22T21:58:00.753 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:3 2026-02-22T21:58:00.753 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:58:00.753 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:58:00.753 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T21:58:00.753 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:3 2026-02-22T21:58:00.753 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:58:00.753 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:58:00.753 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:58:00.753 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=3 2026-02-22T21:58:00.754 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T21:58:00.754 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:58:00.754 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:58:00.754 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=3 2026-02-22T21:58:00.754 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=3 2026-02-22T21:58:00.754 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T21:58:00.754 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:3 2026-02-22T21:58:00.754 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:58:00.754 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:58:00.754 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T21:58:00.754 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:3 2026-02-22T21:58:00.754 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:58:00.755 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:58:00.755 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:58:00.755 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=3 2026-02-22T21:58:00.755 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T21:58:00.755 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:58:00.755 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:58:00.755 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=3 2026-02-22T21:58:00.755 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=3 2026-02-22T21:58:00.755 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T21:58:00.763 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:58:00.763 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:58:00.763 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:58:00.763 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_10 2026-02-22T21:58:00.811 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ return 0 2026-02-22T21:58:00.812 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:58:00.812 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:58:00.812 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep -i Replaying 2026-02-22T21:58:00.813 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: "state": "Replaying" 2026-02-22T21:58:00.814 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_state=started 2026-02-22T21:58:00.814 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test started = started 2026-02-22T21:58:00.814 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:58:00.814 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_10 2026-02-22T21:58:00.814 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T21:58:00.814 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:58:00.814 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:58:00.814 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_10 2026-02-22T21:58:00.814 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = journal ']' 2026-02-22T21:58:00.814 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T21:58:00.814 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ mirror_image_snapshot cluster2 mirror image_10 2026-02-22T21:58:00.814 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:58:00.814 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:58:00.814 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_10 2026-02-22T21:58:00.814 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_10 2026-02-22T21:58:01.573 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Snapshot ID: 104 2026-02-22T21:58:01.579 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_10 2026-02-22T21:58:01.579 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T21:58:01.579 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:58:01.579 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:58:01.579 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_10 2026-02-22T21:58:01.579 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster2-mirror-image_10.status 2026-02-22T21:58:01.579 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster2-mirror-image_10.status 2026-02-22T21:58:01.579 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T21:58:01.580 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_log=/tmp/tmp.j44LZPowal/cluster2-mirror-image_10.status 2026-02-22T21:58:01.581 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster1-mirror-image_10.status 2026-02-22T21:58:01.581 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster1-mirror-image_10.status 2026-02-22T21:58:01.581 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T21:58:01.582 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_status_log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_10.status 2026-02-22T21:58:01.582 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_10 /tmp/tmp.j44LZPowal/cluster2-mirror-image_10.status 2026-02-22T21:58:01.582 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:58:01.582 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:58:01.582 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_10 2026-02-22T21:58:01.582 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster2-mirror-image_10.status 2026-02-22T21:58:01.583 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_10 --format xml 2026-02-22T21:58:01.583 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:58:01.610 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T21:58:01.612 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snapshot_id=104 2026-02-22T21:58:01.612 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:58:01.612 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:58:01.612 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.2 2026-02-22T21:58:01.813 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_10 /tmp/tmp.j44LZPowal/cluster1-mirror-image_10.status 2026-02-22T21:58:01.813 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:58:01.813 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:58:01.813 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_10 2026-02-22T21:58:01.813 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_10.status 2026-02-22T21:58:01.813 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_10 --format xml 2026-02-22T21:58:01.813 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:58:01.843 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:58:01.844 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id=44 2026-02-22T21:58:01.845 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 104 = 44 2026-02-22T21:58:01.845 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:58:01.845 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.4 2026-02-22T21:58:02.246 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_10 /tmp/tmp.j44LZPowal/cluster1-mirror-image_10.status 2026-02-22T21:58:02.246 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:58:02.246 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:58:02.246 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_10 2026-02-22T21:58:02.246 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_10.status 2026-02-22T21:58:02.247 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_10 --format xml 2026-02-22T21:58:02.247 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:58:02.277 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:58:02.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id=44 2026-02-22T21:58:02.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 104 = 44 2026-02-22T21:58:02.279 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:58:02.279 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.8 2026-02-22T21:58:03.079 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_10 /tmp/tmp.j44LZPowal/cluster1-mirror-image_10.status 2026-02-22T21:58:03.079 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:58:03.079 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:58:03.079 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_10 2026-02-22T21:58:03.079 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_10.status 2026-02-22T21:58:03.079 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_10 --format xml 2026-02-22T21:58:03.079 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:58:03.109 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:58:03.109 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:58:03.110 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T21:58:03.110 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T21:58:03.110 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T21:58:03.111 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T21:58:03.111 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 104 = '' 2026-02-22T21:58:03.111 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:58:03.111 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1.6 2026-02-22T21:58:04.711 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_10 /tmp/tmp.j44LZPowal/cluster1-mirror-image_10.status 2026-02-22T21:58:04.712 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:58:04.712 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:58:04.712 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_10 2026-02-22T21:58:04.712 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_10.status 2026-02-22T21:58:04.712 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_10 --format xml 2026-02-22T21:58:04.712 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:58:04.744 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:58:04.746 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id=104 2026-02-22T21:58:04.746 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 104 = 104 2026-02-22T21:58:04.746 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:58:04.746 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snap_present cluster1 mirror image_10 snap 2026-02-22T21:58:04.746 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:58:04.747 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:58:04.747 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_10 2026-02-22T21:58:04.747 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T21:58:04.747 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T21:58:04.747 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T21:58:04.747 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T21:58:05.747 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 info mirror/image_10@snap 2026-02-22T21:58:05.774 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_10': 2026-02-22T21:58:05.774 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 128 MiB in 32 objects 2026-02-22T21:58:05.774 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T21:58:05.774 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 2 2026-02-22T21:58:05.774 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 11bd47f51a57 2026-02-22T21:58:05.774 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.11bd47f51a57 2026-02-22T21:58:05.774 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T21:58:05.774 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock, non-primary 2026-02-22T21:58:05.774 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T21:58:05.775 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T21:58:05.775 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:49:22 2026 2026-02-22T21:58:05.775 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:49:22 2026 2026-02-22T21:58:05.775 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:49:22 2026 2026-02-22T21:58:05.775 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: protected: False 2026-02-22T21:58:05.775 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T21:58:05.775 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T21:58:05.775 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: 0f526ee1-bae4-4f17-9197-c721da6ab0c9 2026-02-22T21:58:05.775 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: false 2026-02-22T21:58:05.776 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:58:05.777 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ compare_image_snaps mirror image_10 snap 2026-02-22T21:58:05.777 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:58:05.777 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_10 2026-02-22T21:58:05.777 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T21:58:05.777 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local ret=0 2026-02-22T21:58:05.777 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local rmt_export=/tmp/tmp.j44LZPowal/cluster2-mirror-image_10.export 2026-02-22T21:58:05.777 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local loc_export=/tmp/tmp.j44LZPowal/cluster1-mirror-image_10.export 2026-02-22T21:58:05.777 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-image_10.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_10.export 2026-02-22T21:58:05.777 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror export image_10@snap /tmp/tmp.j44LZPowal/cluster2-mirror-image_10.export 2026-02-22T21:58:05.883 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T21:58:05.891 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 -p mirror export image_10@snap /tmp/tmp.j44LZPowal/cluster1-mirror-image_10.export 2026-02-22T21:58:06.013 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T21:58:06.021 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ cmp /tmp/tmp.j44LZPowal/cluster2-mirror-image_10.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_10.export 2026-02-22T21:58:06.070 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-image_10.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_10.export 2026-02-22T21:58:06.091 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.0 2026-02-22T21:58:06.091 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Sent signal 15 2026-02-22T21:58:06.091 INFO:tasks.rbd_mirror_thrash:waiting for 8.81156717602384 secs before reviving daemons 2026-02-22T21:58:06.092 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial026.stderr:2026-02-22T21:58:06.089+0000 7f478cf58640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 (PID: 53826) UID: 1000 2026-02-22T21:58:06.107 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:58:06.107 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T21:58:06.107 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_11 2026-02-22T21:58:06.107 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_snap cluster2 mirror image_11 snap 2026-02-22T21:58:06.107 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:58:06.107 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:58:06.107 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_11 2026-02-22T21:58:06.107 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T21:58:06.107 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror snap create image_11@snap --debug-rbd=20 --debug-journaler=20 2026-02-22T21:58:07.169 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_started cluster1 mirror image_11 2026-02-22T21:58:07.169 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:58:07.169 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:58:07.169 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_11 2026-02-22T21:58:07.169 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_state cluster1 mirror image_11 started 2026-02-22T21:58:07.169 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:58:07.169 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:58:07.169 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_11 2026-02-22T21:58:07.169 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=started 2026-02-22T21:58:07.169 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T21:58:07.169 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:58:07.170 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T21:58:08.171 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_replay_state cluster1 mirror image_11 started 2026-02-22T21:58:08.171 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:58:08.171 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:58:08.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_11 2026-02-22T21:58:08.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=started 2026-02-22T21:58:08.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_result 2026-02-22T21:58:08.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=stopped 2026-02-22T21:58:08.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_11 2026-02-22T21:58:08.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_instance=cluster1 2026-02-22T21:58:08.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ grep -i state 2026-02-22T21:58:08.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:58:08.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T21:58:08.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance=cluster1 2026-02-22T21:58:08.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local loop_instance 2026-02-22T21:58:08.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:58:08.173 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sleep 0 2026-02-22T21:58:08.175 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:58:08.175 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ seq 0 3 2026-02-22T21:58:08.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:58:08.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_11 2026-02-22T21:58:08.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:0 2026-02-22T21:58:08.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:58:08.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:58:08.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:58:08.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:0 2026-02-22T21:58:08.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:58:08.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:58:08.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:58:08.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=0 2026-02-22T21:58:08.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:58:08.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:58:08.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:58:08.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=0 2026-02-22T21:58:08.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T21:58:08.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:58:08.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:58:08.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:58:08.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:58:08.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:58:08.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:0 2026-02-22T21:58:08.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:58:08.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:58:08.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:58:08.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T21:58:08.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:58:08.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:58:08.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:58:08.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=0 2026-02-22T21:58:08.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=0 2026-02-22T21:58:08.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:58:08.186 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:58:08.186 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:58:08.186 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:58:08.186 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_11 2026-02-22T21:58:08.223 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-22T21:58:08.226 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:58:08.226 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_11 2026-02-22T21:58:08.226 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:1 2026-02-22T21:58:08.226 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:58:08.227 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:58:08.227 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T21:58:08.227 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:1 2026-02-22T21:58:08.227 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:58:08.227 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:58:08.227 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:58:08.227 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=1 2026-02-22T21:58:08.227 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T21:58:08.227 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:58:08.227 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:58:08.227 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=1 2026-02-22T21:58:08.227 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T21:58:08.228 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T21:58:08.228 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:1 2026-02-22T21:58:08.228 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:58:08.228 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:58:08.228 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T21:58:08.228 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:1 2026-02-22T21:58:08.228 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:58:08.228 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:58:08.228 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:58:08.228 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T21:58:08.228 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T21:58:08.228 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:58:08.229 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:58:08.229 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=1 2026-02-22T21:58:08.229 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=1 2026-02-22T21:58:08.229 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T21:58:08.236 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:58:08.236 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:58:08.236 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:58:08.237 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_11 2026-02-22T21:58:08.281 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_11'] 2026-02-22T21:58:08.281 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:58:08.281 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T21:58:08.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:58:08.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_11 2026-02-22T21:58:08.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:2 2026-02-22T21:58:08.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:58:08.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:58:08.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T21:58:08.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:2 2026-02-22T21:58:08.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:58:08.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:58:08.285 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:58:08.285 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=2 2026-02-22T21:58:08.285 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T21:58:08.285 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:58:08.285 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:58:08.285 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=2 2026-02-22T21:58:08.285 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T21:58:08.285 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T21:58:08.285 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:2 2026-02-22T21:58:08.285 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:58:08.285 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:58:08.285 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T21:58:08.286 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:2 2026-02-22T21:58:08.286 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:58:08.286 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:58:08.286 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:58:08.286 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T21:58:08.286 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T21:58:08.286 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:58:08.286 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:58:08.286 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=2 2026-02-22T21:58:08.286 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=2 2026-02-22T21:58:08.286 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T21:58:08.296 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:58:08.296 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:58:08.296 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:58:08.296 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_11 2026-02-22T21:58:08.345 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ return 0 2026-02-22T21:58:08.345 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:58:08.346 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:58:08.346 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep -i Replaying 2026-02-22T21:58:08.347 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: "state": "Replaying" 2026-02-22T21:58:08.348 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_state=started 2026-02-22T21:58:08.348 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test started = started 2026-02-22T21:58:08.348 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:58:08.348 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_11 2026-02-22T21:58:08.348 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T21:58:08.349 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:58:08.349 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:58:08.349 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_11 2026-02-22T21:58:08.349 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = journal ']' 2026-02-22T21:58:08.349 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T21:58:08.349 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ mirror_image_snapshot cluster2 mirror image_11 2026-02-22T21:58:08.349 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:58:08.349 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:58:08.349 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_11 2026-02-22T21:58:08.349 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_11 2026-02-22T21:58:09.170 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Snapshot ID: 106 2026-02-22T21:58:09.175 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_11 2026-02-22T21:58:09.175 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T21:58:09.175 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:58:09.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:58:09.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_11 2026-02-22T21:58:09.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster2-mirror-image_11.status 2026-02-22T21:58:09.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster2-mirror-image_11.status 2026-02-22T21:58:09.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T21:58:09.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_log=/tmp/tmp.j44LZPowal/cluster2-mirror-image_11.status 2026-02-22T21:58:09.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster1-mirror-image_11.status 2026-02-22T21:58:09.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster1-mirror-image_11.status 2026-02-22T21:58:09.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T21:58:09.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_status_log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_11.status 2026-02-22T21:58:09.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_11 /tmp/tmp.j44LZPowal/cluster2-mirror-image_11.status 2026-02-22T21:58:09.180 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:58:09.180 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:58:09.180 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_11 2026-02-22T21:58:09.180 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster2-mirror-image_11.status 2026-02-22T21:58:09.180 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_11 --format xml 2026-02-22T21:58:09.180 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:58:09.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T21:58:09.209 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snapshot_id=106 2026-02-22T21:58:09.209 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:58:09.209 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:58:09.209 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.2 2026-02-22T21:58:09.410 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_11 /tmp/tmp.j44LZPowal/cluster1-mirror-image_11.status 2026-02-22T21:58:09.410 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:58:09.410 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:58:09.410 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_11 2026-02-22T21:58:09.410 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_11.status 2026-02-22T21:58:09.410 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_11 --format xml 2026-02-22T21:58:09.410 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:58:09.439 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:58:09.440 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:58:09.441 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T21:58:09.441 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T21:58:09.441 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T21:58:09.442 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T21:58:09.442 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 106 = '' 2026-02-22T21:58:09.442 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:58:09.442 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.4 2026-02-22T21:58:09.843 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_11 /tmp/tmp.j44LZPowal/cluster1-mirror-image_11.status 2026-02-22T21:58:09.843 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:58:09.843 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:58:09.843 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_11 2026-02-22T21:58:09.843 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_11.status 2026-02-22T21:58:09.843 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_11 --format xml 2026-02-22T21:58:09.843 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:58:09.873 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:58:09.874 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:58:09.875 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T21:58:09.875 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T21:58:09.875 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T21:58:09.875 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T21:58:09.876 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 106 = '' 2026-02-22T21:58:09.876 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:58:09.876 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.8 2026-02-22T21:58:10.677 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_11 /tmp/tmp.j44LZPowal/cluster1-mirror-image_11.status 2026-02-22T21:58:10.678 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:58:10.678 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:58:10.678 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_11 2026-02-22T21:58:10.678 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_11.status 2026-02-22T21:58:10.678 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_11 --format xml 2026-02-22T21:58:10.678 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:58:10.707 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:58:10.709 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id=106 2026-02-22T21:58:10.709 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 106 = 106 2026-02-22T21:58:10.709 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:58:10.709 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snap_present cluster1 mirror image_11 snap 2026-02-22T21:58:10.709 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:58:10.709 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:58:10.710 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_11 2026-02-22T21:58:10.710 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T21:58:10.710 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T21:58:10.710 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T21:58:10.710 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T21:58:11.710 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 info mirror/image_11@snap 2026-02-22T21:58:11.740 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_11': 2026-02-22T21:58:11.740 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 128 MiB in 32 objects 2026-02-22T21:58:11.741 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T21:58:11.741 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 2 2026-02-22T21:58:11.741 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 1227bdb42f78 2026-02-22T21:58:11.741 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.1227bdb42f78 2026-02-22T21:58:11.741 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T21:58:11.741 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock, non-primary 2026-02-22T21:58:11.741 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T21:58:11.741 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T21:58:11.741 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:49:35 2026 2026-02-22T21:58:11.742 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:49:35 2026 2026-02-22T21:58:11.742 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:49:35 2026 2026-02-22T21:58:11.742 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: protected: False 2026-02-22T21:58:11.742 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T21:58:11.742 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T21:58:11.743 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: 69c86d55-4653-4f0e-a6ff-bc5fe92ac722 2026-02-22T21:58:11.743 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: false 2026-02-22T21:58:11.743 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:58:11.743 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ compare_image_snaps mirror image_11 snap 2026-02-22T21:58:11.743 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:58:11.743 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_11 2026-02-22T21:58:11.743 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T21:58:11.743 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local ret=0 2026-02-22T21:58:11.743 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local rmt_export=/tmp/tmp.j44LZPowal/cluster2-mirror-image_11.export 2026-02-22T21:58:11.744 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local loc_export=/tmp/tmp.j44LZPowal/cluster1-mirror-image_11.export 2026-02-22T21:58:11.744 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-image_11.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_11.export 2026-02-22T21:58:11.744 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror export image_11@snap /tmp/tmp.j44LZPowal/cluster2-mirror-image_11.export 2026-02-22T21:58:11.847 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T21:58:11.855 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 -p mirror export image_11@snap /tmp/tmp.j44LZPowal/cluster1-mirror-image_11.export 2026-02-22T21:58:12.163 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T21:58:12.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ cmp /tmp/tmp.j44LZPowal/cluster2-mirror-image_11.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_11.export 2026-02-22T21:58:12.220 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-image_11.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_11.export 2026-02-22T21:58:12.255 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:58:12.255 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T21:58:12.255 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_12 2026-02-22T21:58:12.255 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_snap cluster2 mirror image_12 snap 2026-02-22T21:58:12.255 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:58:12.256 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:58:12.256 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_12 2026-02-22T21:58:12.256 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T21:58:12.256 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror snap create image_12@snap --debug-rbd=20 --debug-journaler=20 2026-02-22T21:58:14.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_started cluster1 mirror image_12 2026-02-22T21:58:14.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:58:14.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:58:14.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_12 2026-02-22T21:58:14.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_state cluster1 mirror image_12 started 2026-02-22T21:58:14.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:58:14.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:58:14.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_12 2026-02-22T21:58:14.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=started 2026-02-22T21:58:14.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T21:58:14.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:58:14.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T21:58:14.903 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.0 2026-02-22T21:58:14.903 INFO:teuthology.orchestra.run:waiting for 600 2026-02-22T21:58:14.904 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.0 2026-02-22T21:58:14.904 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Restarting daemon 2026-02-22T21:58:14.904 DEBUG:teuthology.orchestra.run.trial026:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 2026-02-22T21:58:14.907 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Started 2026-02-22T21:58:14.907 INFO:tasks.rbd_mirror_thrash:waiting for 107.37320508518881 secs before thrashing 2026-02-22T21:58:15.174 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_replay_state cluster1 mirror image_12 started 2026-02-22T21:58:15.174 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:58:15.174 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:58:15.174 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_12 2026-02-22T21:58:15.174 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=started 2026-02-22T21:58:15.174 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_result 2026-02-22T21:58:15.174 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=stopped 2026-02-22T21:58:15.175 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_12 2026-02-22T21:58:15.175 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ grep -i state 2026-02-22T21:58:15.175 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_instance=cluster1 2026-02-22T21:58:15.175 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:58:15.175 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T21:58:15.175 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance=cluster1 2026-02-22T21:58:15.175 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local loop_instance 2026-02-22T21:58:15.175 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:58:15.175 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sleep 0 2026-02-22T21:58:15.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:58:15.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ seq 0 3 2026-02-22T21:58:15.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:58:15.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_12 2026-02-22T21:58:15.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:0 2026-02-22T21:58:15.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:58:15.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:58:15.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:58:15.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:0 2026-02-22T21:58:15.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:58:15.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:58:15.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:58:15.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=0 2026-02-22T21:58:15.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:58:15.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:58:15.180 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:58:15.180 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=0 2026-02-22T21:58:15.180 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T21:58:15.180 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:58:15.180 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:58:15.180 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:58:15.180 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:58:15.180 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:58:15.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:0 2026-02-22T21:58:15.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:58:15.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:58:15.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:58:15.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T21:58:15.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:58:15.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:58:15.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:58:15.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=0 2026-02-22T21:58:15.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=0 2026-02-22T21:58:15.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:58:15.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:58:15.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:58:15.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:58:15.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_12 2026-02-22T21:58:15.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_12'] 2026-02-22T21:58:15.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:58:15.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T21:58:15.235 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:58:15.235 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_12 2026-02-22T21:58:15.235 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:1 2026-02-22T21:58:15.235 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:58:15.235 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:58:15.235 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T21:58:15.235 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:1 2026-02-22T21:58:15.235 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:58:15.235 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:58:15.236 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:58:15.236 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=1 2026-02-22T21:58:15.236 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T21:58:15.236 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:58:15.236 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:58:15.236 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=1 2026-02-22T21:58:15.236 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T21:58:15.236 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T21:58:15.236 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:1 2026-02-22T21:58:15.236 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:58:15.236 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:58:15.236 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T21:58:15.237 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:1 2026-02-22T21:58:15.237 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:58:15.237 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:58:15.237 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:58:15.237 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T21:58:15.237 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T21:58:15.237 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:58:15.237 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:58:15.237 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=1 2026-02-22T21:58:15.237 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=1 2026-02-22T21:58:15.237 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T21:58:15.245 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:58:15.245 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:58:15.246 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:58:15.246 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_12 2026-02-22T21:58:15.290 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_12'] 2026-02-22T21:58:15.291 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:58:15.291 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T21:58:15.294 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:58:15.294 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_12 2026-02-22T21:58:15.294 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:2 2026-02-22T21:58:15.294 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:58:15.295 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:58:15.295 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T21:58:15.295 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:2 2026-02-22T21:58:15.295 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:58:15.295 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:58:15.295 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:58:15.295 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=2 2026-02-22T21:58:15.295 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T21:58:15.295 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:58:15.295 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:58:15.295 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=2 2026-02-22T21:58:15.296 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T21:58:15.296 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T21:58:15.296 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:2 2026-02-22T21:58:15.296 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:58:15.296 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:58:15.296 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T21:58:15.296 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:2 2026-02-22T21:58:15.296 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:58:15.296 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:58:15.296 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:58:15.296 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T21:58:15.296 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T21:58:15.297 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:58:15.297 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:58:15.297 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=2 2026-02-22T21:58:15.297 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=2 2026-02-22T21:58:15.297 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T21:58:15.305 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:58:15.305 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:58:15.305 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:58:15.305 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_12 2026-02-22T21:58:15.356 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ return 0 2026-02-22T21:58:15.357 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:58:15.357 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:58:15.357 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep -i Replaying 2026-02-22T21:58:15.358 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: "state": "Replaying" 2026-02-22T21:58:15.359 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_state=started 2026-02-22T21:58:15.359 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test started = started 2026-02-22T21:58:15.359 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:58:15.359 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_12 2026-02-22T21:58:15.359 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T21:58:15.360 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:58:15.360 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:58:15.360 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_12 2026-02-22T21:58:15.360 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = journal ']' 2026-02-22T21:58:15.360 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T21:58:15.360 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ mirror_image_snapshot cluster2 mirror image_12 2026-02-22T21:58:15.360 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:58:15.360 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:58:15.360 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_12 2026-02-22T21:58:15.360 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_12 2026-02-22T21:58:16.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Snapshot ID: 108 2026-02-22T21:58:16.184 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_12 2026-02-22T21:58:16.184 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T21:58:16.185 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:58:16.185 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:58:16.185 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_12 2026-02-22T21:58:16.185 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster2-mirror-image_12.status 2026-02-22T21:58:16.185 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster2-mirror-image_12.status 2026-02-22T21:58:16.185 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T21:58:16.186 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_log=/tmp/tmp.j44LZPowal/cluster2-mirror-image_12.status 2026-02-22T21:58:16.186 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster1-mirror-image_12.status 2026-02-22T21:58:16.186 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster1-mirror-image_12.status 2026-02-22T21:58:16.187 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T21:58:16.188 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_status_log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_12.status 2026-02-22T21:58:16.188 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_12 /tmp/tmp.j44LZPowal/cluster2-mirror-image_12.status 2026-02-22T21:58:16.188 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:58:16.188 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:58:16.188 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_12 2026-02-22T21:58:16.188 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster2-mirror-image_12.status 2026-02-22T21:58:16.188 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_12 --format xml 2026-02-22T21:58:16.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:58:16.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T21:58:16.218 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snapshot_id=108 2026-02-22T21:58:16.218 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:58:16.218 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:58:16.218 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.2 2026-02-22T21:58:16.419 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_12 /tmp/tmp.j44LZPowal/cluster1-mirror-image_12.status 2026-02-22T21:58:16.419 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:58:16.420 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:58:16.420 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_12 2026-02-22T21:58:16.420 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_12.status 2026-02-22T21:58:16.420 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_12 --format xml 2026-02-22T21:58:16.420 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:58:16.450 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:58:16.451 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:58:16.451 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T21:58:16.452 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T21:58:16.452 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T21:58:16.452 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T21:58:16.453 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 108 = '' 2026-02-22T21:58:16.453 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:58:16.453 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.4 2026-02-22T21:58:16.853 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_12 /tmp/tmp.j44LZPowal/cluster1-mirror-image_12.status 2026-02-22T21:58:16.853 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:58:16.853 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:58:16.853 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_12 2026-02-22T21:58:16.853 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_12.status 2026-02-22T21:58:16.853 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_12 --format xml 2026-02-22T21:58:16.853 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:58:16.884 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:58:16.884 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:58:16.885 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T21:58:16.885 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T21:58:16.885 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T21:58:16.886 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T21:58:16.886 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 108 = '' 2026-02-22T21:58:16.886 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:58:16.886 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.8 2026-02-22T21:58:17.686 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_12 /tmp/tmp.j44LZPowal/cluster1-mirror-image_12.status 2026-02-22T21:58:17.686 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:58:17.686 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:58:17.686 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_12 2026-02-22T21:58:17.687 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_12.status 2026-02-22T21:58:17.687 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_12 --format xml 2026-02-22T21:58:17.687 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:58:17.720 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:58:17.721 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id=108 2026-02-22T21:58:17.721 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 108 = 108 2026-02-22T21:58:17.721 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:58:17.722 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snap_present cluster1 mirror image_12 snap 2026-02-22T21:58:17.722 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:58:17.722 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:58:17.722 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_12 2026-02-22T21:58:17.722 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T21:58:17.722 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T21:58:17.722 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T21:58:17.722 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T21:58:18.723 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 info mirror/image_12@snap 2026-02-22T21:58:18.755 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_12': 2026-02-22T21:58:18.755 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 128 MiB in 32 objects 2026-02-22T21:58:18.755 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T21:58:18.755 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 2 2026-02-22T21:58:18.755 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 1248fc10bace 2026-02-22T21:58:18.755 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.1248fc10bace 2026-02-22T21:58:18.755 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T21:58:18.755 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock, non-primary 2026-02-22T21:58:18.755 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T21:58:18.756 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T21:58:18.756 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:49:47 2026 2026-02-22T21:58:18.756 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:49:47 2026 2026-02-22T21:58:18.756 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:49:47 2026 2026-02-22T21:58:18.756 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: protected: False 2026-02-22T21:58:18.756 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T21:58:18.756 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T21:58:18.756 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: 82008379-765b-45b9-b3f2-d57783e88ea0 2026-02-22T21:58:18.756 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: false 2026-02-22T21:58:18.757 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:58:18.758 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ compare_image_snaps mirror image_12 snap 2026-02-22T21:58:18.758 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:58:18.758 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_12 2026-02-22T21:58:18.758 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T21:58:18.758 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local ret=0 2026-02-22T21:58:18.758 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local rmt_export=/tmp/tmp.j44LZPowal/cluster2-mirror-image_12.export 2026-02-22T21:58:18.758 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local loc_export=/tmp/tmp.j44LZPowal/cluster1-mirror-image_12.export 2026-02-22T21:58:18.758 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-image_12.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_12.export 2026-02-22T21:58:18.758 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror export image_12@snap /tmp/tmp.j44LZPowal/cluster2-mirror-image_12.export 2026-02-22T21:58:18.856 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T21:58:18.863 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 -p mirror export image_12@snap /tmp/tmp.j44LZPowal/cluster1-mirror-image_12.export 2026-02-22T21:58:18.990 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T21:58:18.998 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ cmp /tmp/tmp.j44LZPowal/cluster2-mirror-image_12.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_12.export 2026-02-22T21:58:19.046 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-image_12.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_12.export 2026-02-22T21:58:19.081 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:58:19.081 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T21:58:19.081 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_13 2026-02-22T21:58:19.082 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_snap cluster2 mirror image_13 snap 2026-02-22T21:58:19.082 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:58:19.082 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:58:19.082 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_13 2026-02-22T21:58:19.082 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T21:58:19.082 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror snap create image_13@snap --debug-rbd=20 --debug-journaler=20 2026-02-22T21:58:20.184 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_started cluster1 mirror image_13 2026-02-22T21:58:20.184 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:58:20.184 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:58:20.185 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_13 2026-02-22T21:58:20.185 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_state cluster1 mirror image_13 started 2026-02-22T21:58:20.185 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:58:20.185 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:58:20.185 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_13 2026-02-22T21:58:20.185 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=started 2026-02-22T21:58:20.185 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T21:58:20.185 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:58:20.186 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T21:58:21.186 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_replay_state cluster1 mirror image_13 started 2026-02-22T21:58:21.186 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:58:21.186 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:58:21.186 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_13 2026-02-22T21:58:21.186 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=started 2026-02-22T21:58:21.187 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_result 2026-02-22T21:58:21.187 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=stopped 2026-02-22T21:58:21.187 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_13 2026-02-22T21:58:21.187 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_instance=cluster1 2026-02-22T21:58:21.187 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:58:21.187 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T21:58:21.187 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance=cluster1 2026-02-22T21:58:21.187 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local loop_instance 2026-02-22T21:58:21.187 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:58:21.187 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ grep -i state 2026-02-22T21:58:21.188 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sleep 0 2026-02-22T21:58:21.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:58:21.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ seq 0 3 2026-02-22T21:58:21.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:58:21.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_13 2026-02-22T21:58:21.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:0 2026-02-22T21:58:21.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:58:21.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:58:21.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:58:21.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:0 2026-02-22T21:58:21.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:58:21.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:58:21.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:58:21.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=0 2026-02-22T21:58:21.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:58:21.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:58:21.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:58:21.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=0 2026-02-22T21:58:21.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T21:58:21.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:58:21.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:58:21.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:58:21.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:58:21.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:58:21.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:0 2026-02-22T21:58:21.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:58:21.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:58:21.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:58:21.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T21:58:21.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:58:21.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:58:21.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:58:21.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=0 2026-02-22T21:58:21.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=0 2026-02-22T21:58:21.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:58:21.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:58:21.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:58:21.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:58:21.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_13 2026-02-22T21:58:21.243 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_13'] 2026-02-22T21:58:21.243 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:58:21.244 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T21:58:21.246 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:58:21.246 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_13 2026-02-22T21:58:21.247 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:1 2026-02-22T21:58:21.247 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:58:21.247 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:58:21.247 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T21:58:21.247 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:1 2026-02-22T21:58:21.247 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:58:21.247 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:58:21.247 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:58:21.248 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=1 2026-02-22T21:58:21.248 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T21:58:21.248 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:58:21.248 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:58:21.248 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=1 2026-02-22T21:58:21.248 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T21:58:21.248 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T21:58:21.248 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:1 2026-02-22T21:58:21.248 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:58:21.248 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:58:21.248 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T21:58:21.248 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:1 2026-02-22T21:58:21.249 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:58:21.249 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:58:21.249 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:58:21.249 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T21:58:21.249 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T21:58:21.249 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:58:21.249 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:58:21.249 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=1 2026-02-22T21:58:21.249 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=1 2026-02-22T21:58:21.249 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T21:58:21.257 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:58:21.257 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:58:21.258 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:58:21.258 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_13 2026-02-22T21:58:21.300 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_13'] 2026-02-22T21:58:21.300 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:58:21.300 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T21:58:21.303 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:58:21.303 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_13 2026-02-22T21:58:21.303 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:2 2026-02-22T21:58:21.304 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:58:21.304 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:58:21.304 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T21:58:21.304 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:2 2026-02-22T21:58:21.304 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:58:21.304 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:58:21.304 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:58:21.305 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=2 2026-02-22T21:58:21.305 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T21:58:21.305 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:58:21.305 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:58:21.305 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=2 2026-02-22T21:58:21.305 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T21:58:21.306 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T21:58:21.306 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:2 2026-02-22T21:58:21.306 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:58:21.306 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:58:21.306 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T21:58:21.306 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:2 2026-02-22T21:58:21.306 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:58:21.307 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:58:21.307 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:58:21.307 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T21:58:21.307 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T21:58:21.307 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:58:21.307 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:58:21.307 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=2 2026-02-22T21:58:21.308 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=2 2026-02-22T21:58:21.308 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T21:58:21.313 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:58:21.313 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:58:21.313 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:58:21.313 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_13 2026-02-22T21:58:21.358 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_13'] 2026-02-22T21:58:21.358 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:58:21.358 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T21:58:21.360 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:58:21.360 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_13 2026-02-22T21:58:21.360 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:3 2026-02-22T21:58:21.360 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:58:21.360 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:58:21.360 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T21:58:21.360 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:3 2026-02-22T21:58:21.360 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:58:21.360 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:58:21.360 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:58:21.360 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=3 2026-02-22T21:58:21.361 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T21:58:21.361 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:58:21.361 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:58:21.361 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=3 2026-02-22T21:58:21.361 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=3 2026-02-22T21:58:21.361 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T21:58:21.361 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:3 2026-02-22T21:58:21.361 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:58:21.361 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:58:21.361 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T21:58:21.361 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:3 2026-02-22T21:58:21.361 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:58:21.361 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:58:21.361 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:58:21.361 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=3 2026-02-22T21:58:21.361 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T21:58:21.361 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:58:21.361 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:58:21.361 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=3 2026-02-22T21:58:21.361 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=3 2026-02-22T21:58:21.361 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T21:58:21.371 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:58:21.371 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:58:21.371 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:58:21.372 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_13 2026-02-22T21:58:21.417 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_13'] 2026-02-22T21:58:21.417 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:58:21.417 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T21:58:21.418 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:58:21.418 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sleep 1 2026-02-22T21:58:22.420 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:58:22.420 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ seq 0 3 2026-02-22T21:58:22.423 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:58:22.424 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_13 2026-02-22T21:58:22.424 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:0 2026-02-22T21:58:22.424 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:58:22.424 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:58:22.424 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:58:22.424 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:0 2026-02-22T21:58:22.424 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:58:22.424 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:58:22.424 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:58:22.424 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=0 2026-02-22T21:58:22.425 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:58:22.425 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:58:22.425 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:58:22.425 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=0 2026-02-22T21:58:22.425 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T21:58:22.425 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:58:22.425 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:58:22.425 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:58:22.425 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:58:22.425 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:58:22.425 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:0 2026-02-22T21:58:22.425 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:58:22.426 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:58:22.426 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:58:22.426 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T21:58:22.426 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:58:22.426 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:58:22.426 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:58:22.426 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=0 2026-02-22T21:58:22.426 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=0 2026-02-22T21:58:22.426 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:58:22.435 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:58:22.435 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:58:22.435 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:58:22.435 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_13 2026-02-22T21:58:22.478 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_13'] 2026-02-22T21:58:22.478 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:58:22.479 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T21:58:22.482 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:58:22.483 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_13 2026-02-22T21:58:22.483 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:1 2026-02-22T21:58:22.483 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:58:22.483 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:58:22.483 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T21:58:22.483 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:1 2026-02-22T21:58:22.483 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:58:22.483 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:58:22.483 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:58:22.484 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=1 2026-02-22T21:58:22.484 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T21:58:22.484 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:58:22.484 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:58:22.484 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=1 2026-02-22T21:58:22.484 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T21:58:22.484 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T21:58:22.484 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:1 2026-02-22T21:58:22.484 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:58:22.484 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:58:22.485 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T21:58:22.485 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:1 2026-02-22T21:58:22.485 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:58:22.485 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:58:22.485 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:58:22.485 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T21:58:22.485 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T21:58:22.485 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:58:22.485 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:58:22.485 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=1 2026-02-22T21:58:22.486 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=1 2026-02-22T21:58:22.486 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T21:58:22.493 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:58:22.493 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:58:22.493 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:58:22.493 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_13 2026-02-22T21:58:22.537 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_13'] 2026-02-22T21:58:22.537 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:58:22.537 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T21:58:22.540 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:58:22.540 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_13 2026-02-22T21:58:22.540 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:2 2026-02-22T21:58:22.540 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:58:22.540 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:58:22.541 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T21:58:22.541 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:2 2026-02-22T21:58:22.541 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:58:22.541 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:58:22.541 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:58:22.541 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=2 2026-02-22T21:58:22.541 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T21:58:22.541 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:58:22.541 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:58:22.541 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=2 2026-02-22T21:58:22.541 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T21:58:22.541 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T21:58:22.542 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:2 2026-02-22T21:58:22.542 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:58:22.542 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:58:22.542 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T21:58:22.542 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:2 2026-02-22T21:58:22.542 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:58:22.542 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:58:22.542 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:58:22.542 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T21:58:22.542 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T21:58:22.542 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:58:22.542 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:58:22.542 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=2 2026-02-22T21:58:22.543 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=2 2026-02-22T21:58:22.543 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T21:58:22.550 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:58:22.551 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:58:22.551 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:58:22.551 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_13 2026-02-22T21:58:22.596 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_13'] 2026-02-22T21:58:22.596 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:58:22.596 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T21:58:22.599 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:58:22.599 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_13 2026-02-22T21:58:22.600 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:3 2026-02-22T21:58:22.600 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:58:22.600 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:58:22.600 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T21:58:22.600 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:3 2026-02-22T21:58:22.600 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:58:22.600 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:58:22.600 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:58:22.600 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=3 2026-02-22T21:58:22.600 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T21:58:22.600 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:58:22.601 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:58:22.601 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=3 2026-02-22T21:58:22.601 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=3 2026-02-22T21:58:22.601 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T21:58:22.601 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:3 2026-02-22T21:58:22.601 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:58:22.601 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:58:22.601 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T21:58:22.601 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:3 2026-02-22T21:58:22.601 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:58:22.601 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:58:22.601 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:58:22.602 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=3 2026-02-22T21:58:22.602 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T21:58:22.602 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:58:22.602 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:58:22.602 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=3 2026-02-22T21:58:22.602 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=3 2026-02-22T21:58:22.602 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T21:58:22.609 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:58:22.609 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:58:22.609 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:58:22.610 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_13 2026-02-22T21:58:22.654 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_13'] 2026-02-22T21:58:22.654 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:58:22.654 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T21:58:22.655 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:58:22.655 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sleep 2 2026-02-22T21:58:24.656 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:58:24.657 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ seq 0 3 2026-02-22T21:58:24.659 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:58:24.659 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_13 2026-02-22T21:58:24.659 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:0 2026-02-22T21:58:24.660 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:58:24.660 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:58:24.660 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:58:24.660 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:0 2026-02-22T21:58:24.660 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:58:24.660 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:58:24.660 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:58:24.660 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=0 2026-02-22T21:58:24.660 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:58:24.660 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:58:24.660 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:58:24.661 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=0 2026-02-22T21:58:24.661 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T21:58:24.661 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:58:24.661 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:58:24.661 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:58:24.661 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:58:24.661 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:58:24.661 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:0 2026-02-22T21:58:24.661 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:58:24.661 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:58:24.661 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:58:24.662 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T21:58:24.662 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:58:24.662 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:58:24.662 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:58:24.662 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=0 2026-02-22T21:58:24.662 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=0 2026-02-22T21:58:24.662 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:58:24.672 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:58:24.672 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:58:24.672 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:58:24.672 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_13 2026-02-22T21:58:24.714 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_13'] 2026-02-22T21:58:24.714 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:58:24.714 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T21:58:24.717 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:58:24.717 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_13 2026-02-22T21:58:24.717 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:1 2026-02-22T21:58:24.717 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:58:24.717 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:58:24.718 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T21:58:24.718 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:1 2026-02-22T21:58:24.718 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:58:24.718 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:58:24.718 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:58:24.718 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=1 2026-02-22T21:58:24.718 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T21:58:24.718 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:58:24.718 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:58:24.718 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=1 2026-02-22T21:58:24.718 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T21:58:24.718 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T21:58:24.719 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:1 2026-02-22T21:58:24.719 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:58:24.719 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:58:24.719 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T21:58:24.719 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:1 2026-02-22T21:58:24.719 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:58:24.719 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:58:24.719 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:58:24.719 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T21:58:24.719 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T21:58:24.719 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:58:24.719 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:58:24.719 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=1 2026-02-22T21:58:24.720 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=1 2026-02-22T21:58:24.720 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T21:58:24.727 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:58:24.728 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:58:24.728 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:58:24.728 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_13 2026-02-22T21:58:24.772 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_13'] 2026-02-22T21:58:24.772 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:58:24.772 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T21:58:24.775 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:58:24.776 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_13 2026-02-22T21:58:24.776 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:2 2026-02-22T21:58:24.776 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:58:24.776 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:58:24.776 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T21:58:24.776 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:2 2026-02-22T21:58:24.776 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:58:24.776 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:58:24.776 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:58:24.776 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=2 2026-02-22T21:58:24.776 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T21:58:24.776 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:58:24.776 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:58:24.776 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=2 2026-02-22T21:58:24.777 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T21:58:24.777 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T21:58:24.777 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:2 2026-02-22T21:58:24.777 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:58:24.777 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:58:24.777 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T21:58:24.777 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:2 2026-02-22T21:58:24.777 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:58:24.777 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:58:24.777 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:58:24.777 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T21:58:24.777 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T21:58:24.777 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:58:24.777 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:58:24.777 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=2 2026-02-22T21:58:24.778 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=2 2026-02-22T21:58:24.778 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T21:58:24.786 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:58:24.786 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:58:24.786 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:58:24.786 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_13 2026-02-22T21:58:24.833 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_13'] 2026-02-22T21:58:24.833 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:58:24.833 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T21:58:24.837 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:58:24.837 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_13 2026-02-22T21:58:24.837 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:3 2026-02-22T21:58:24.837 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:58:24.837 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:58:24.837 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T21:58:24.837 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:3 2026-02-22T21:58:24.837 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:58:24.837 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:58:24.838 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:58:24.838 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=3 2026-02-22T21:58:24.838 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T21:58:24.838 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:58:24.838 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:58:24.838 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=3 2026-02-22T21:58:24.838 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=3 2026-02-22T21:58:24.838 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T21:58:24.838 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:3 2026-02-22T21:58:24.838 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:58:24.838 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:58:24.838 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T21:58:24.839 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:3 2026-02-22T21:58:24.839 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:58:24.839 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:58:24.839 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:58:24.839 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=3 2026-02-22T21:58:24.839 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T21:58:24.839 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:58:24.839 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:58:24.839 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=3 2026-02-22T21:58:24.839 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=3 2026-02-22T21:58:24.840 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T21:58:24.847 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:58:24.848 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:58:24.848 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:58:24.848 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_13 2026-02-22T21:58:24.894 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_13'] 2026-02-22T21:58:24.894 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:58:24.894 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T21:58:24.895 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:58:24.895 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sleep 4 2026-02-22T21:58:28.896 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:58:28.896 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ seq 0 3 2026-02-22T21:58:28.898 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:58:28.898 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_13 2026-02-22T21:58:28.898 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:0 2026-02-22T21:58:28.898 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:58:28.898 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:58:28.898 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:58:28.898 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:0 2026-02-22T21:58:28.898 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:58:28.898 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:58:28.898 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:58:28.898 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=0 2026-02-22T21:58:28.898 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:58:28.898 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:58:28.898 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:58:28.899 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=0 2026-02-22T21:58:28.899 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T21:58:28.899 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:58:28.899 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:58:28.899 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:58:28.899 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:58:28.899 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:58:28.899 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:0 2026-02-22T21:58:28.899 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:58:28.899 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:58:28.899 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:58:28.899 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T21:58:28.899 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:58:28.899 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:58:28.899 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:58:28.899 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=0 2026-02-22T21:58:28.899 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=0 2026-02-22T21:58:28.899 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:58:28.911 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:58:28.912 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:58:28.912 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:58:28.912 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_13 2026-02-22T21:58:28.954 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_13'] 2026-02-22T21:58:28.954 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:58:28.954 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T21:58:28.955 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:58:28.956 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_13 2026-02-22T21:58:28.956 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:1 2026-02-22T21:58:28.956 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:58:28.956 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:58:28.956 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T21:58:28.956 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:1 2026-02-22T21:58:28.956 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:58:28.956 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:58:28.956 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:58:28.956 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=1 2026-02-22T21:58:28.956 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T21:58:28.956 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:58:28.956 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:58:28.956 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=1 2026-02-22T21:58:28.956 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T21:58:28.956 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T21:58:28.956 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:1 2026-02-22T21:58:28.956 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:58:28.956 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:58:28.956 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T21:58:28.956 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:1 2026-02-22T21:58:28.956 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:58:28.956 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:58:28.956 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:58:28.956 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T21:58:28.957 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T21:58:28.957 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:58:28.957 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:58:28.957 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=1 2026-02-22T21:58:28.957 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=1 2026-02-22T21:58:28.957 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T21:58:28.968 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:58:28.968 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:58:28.968 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:58:28.968 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_13 2026-02-22T21:58:29.013 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_13'] 2026-02-22T21:58:29.013 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:58:29.013 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T21:58:29.015 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:58:29.015 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_13 2026-02-22T21:58:29.015 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:2 2026-02-22T21:58:29.015 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:58:29.015 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:58:29.015 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T21:58:29.015 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:2 2026-02-22T21:58:29.015 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:58:29.015 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:58:29.015 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:58:29.015 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=2 2026-02-22T21:58:29.016 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T21:58:29.016 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:58:29.016 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:58:29.016 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=2 2026-02-22T21:58:29.016 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T21:58:29.016 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T21:58:29.016 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:2 2026-02-22T21:58:29.016 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:58:29.016 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:58:29.016 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T21:58:29.016 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:2 2026-02-22T21:58:29.016 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:58:29.016 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:58:29.016 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:58:29.016 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T21:58:29.016 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T21:58:29.016 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:58:29.016 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:58:29.016 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=2 2026-02-22T21:58:29.016 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=2 2026-02-22T21:58:29.016 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T21:58:29.027 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:58:29.027 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:58:29.027 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:58:29.027 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_13 2026-02-22T21:58:29.072 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_13'] 2026-02-22T21:58:29.072 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:58:29.073 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T21:58:29.076 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:58:29.077 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_13 2026-02-22T21:58:29.077 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:3 2026-02-22T21:58:29.077 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:58:29.077 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:58:29.077 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T21:58:29.077 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:3 2026-02-22T21:58:29.077 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:58:29.078 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:58:29.078 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:58:29.078 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=3 2026-02-22T21:58:29.078 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T21:58:29.078 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:58:29.078 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:58:29.078 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=3 2026-02-22T21:58:29.078 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=3 2026-02-22T21:58:29.079 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T21:58:29.079 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:3 2026-02-22T21:58:29.079 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:58:29.079 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:58:29.079 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T21:58:29.079 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:3 2026-02-22T21:58:29.079 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:58:29.079 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:58:29.079 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:58:29.080 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=3 2026-02-22T21:58:29.080 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T21:58:29.080 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:58:29.080 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:58:29.080 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=3 2026-02-22T21:58:29.080 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=3 2026-02-22T21:58:29.080 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T21:58:29.086 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:58:29.087 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:58:29.087 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:58:29.087 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_13 2026-02-22T21:58:29.131 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_13'] 2026-02-22T21:58:29.132 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:58:29.132 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T21:58:29.133 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:58:29.133 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sleep 8 2026-02-22T21:58:30.550 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.0 2026-02-22T21:58:30.550 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Sent signal 15 2026-02-22T21:58:30.551 INFO:tasks.rbd_mirror_thrash:waiting for 4.025800988271595 secs before reviving daemons 2026-02-22T21:58:30.551 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial026.stderr:2026-02-22T21:58:30.550+0000 7fb78e193640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 (PID: 58927) UID: 1000 2026-02-22T21:58:34.578 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.0 2026-02-22T21:58:34.578 INFO:teuthology.orchestra.run:waiting for 600 2026-02-22T21:58:34.579 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.0 2026-02-22T21:58:34.579 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Restarting daemon 2026-02-22T21:58:34.579 DEBUG:teuthology.orchestra.run.trial026:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 2026-02-22T21:58:34.581 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Started 2026-02-22T21:58:34.582 INFO:tasks.rbd_mirror_thrash:waiting for 95.2076595806316 secs before thrashing 2026-02-22T21:58:37.134 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:58:37.134 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ seq 0 3 2026-02-22T21:58:37.137 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:58:37.137 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_13 2026-02-22T21:58:37.137 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:0 2026-02-22T21:58:37.137 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:58:37.138 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:58:37.138 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:58:37.138 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:0 2026-02-22T21:58:37.138 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:58:37.138 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:58:37.138 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:58:37.138 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=0 2026-02-22T21:58:37.138 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:58:37.138 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:58:37.138 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:58:37.138 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=0 2026-02-22T21:58:37.138 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T21:58:37.138 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:58:37.138 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:58:37.139 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:58:37.139 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:58:37.139 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:58:37.139 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:0 2026-02-22T21:58:37.139 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:58:37.139 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:58:37.139 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:58:37.139 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T21:58:37.139 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:58:37.139 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:58:37.139 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:58:37.139 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=0 2026-02-22T21:58:37.139 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=0 2026-02-22T21:58:37.139 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:58:37.150 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:58:37.151 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:58:37.151 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:58:37.151 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_13 2026-02-22T21:58:37.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_13'] 2026-02-22T21:58:37.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:58:37.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T21:58:37.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:58:37.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_13 2026-02-22T21:58:37.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:1 2026-02-22T21:58:37.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:58:37.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:58:37.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T21:58:37.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:1 2026-02-22T21:58:37.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:58:37.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:58:37.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:58:37.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=1 2026-02-22T21:58:37.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T21:58:37.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:58:37.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:58:37.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=1 2026-02-22T21:58:37.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T21:58:37.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T21:58:37.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:1 2026-02-22T21:58:37.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:58:37.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:58:37.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T21:58:37.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:1 2026-02-22T21:58:37.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:58:37.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:58:37.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:58:37.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T21:58:37.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T21:58:37.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:58:37.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:58:37.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=1 2026-02-22T21:58:37.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=1 2026-02-22T21:58:37.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T21:58:37.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:58:37.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:58:37.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:58:37.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_13 2026-02-22T21:58:37.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_13'] 2026-02-22T21:58:37.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:58:37.254 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T21:58:37.257 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:58:37.257 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_13 2026-02-22T21:58:37.257 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:2 2026-02-22T21:58:37.257 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:58:37.257 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:58:37.258 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T21:58:37.258 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:2 2026-02-22T21:58:37.258 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:58:37.258 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:58:37.258 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:58:37.258 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=2 2026-02-22T21:58:37.258 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T21:58:37.258 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:58:37.258 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:58:37.258 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=2 2026-02-22T21:58:37.258 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T21:58:37.258 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T21:58:37.258 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:2 2026-02-22T21:58:37.258 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:58:37.258 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:58:37.258 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T21:58:37.258 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:2 2026-02-22T21:58:37.258 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:58:37.258 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:58:37.258 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:58:37.259 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T21:58:37.259 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T21:58:37.259 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:58:37.259 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:58:37.259 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=2 2026-02-22T21:58:37.259 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=2 2026-02-22T21:58:37.259 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T21:58:37.267 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:58:37.267 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:58:37.267 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:58:37.267 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_13 2026-02-22T21:58:37.312 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_13'] 2026-02-22T21:58:37.313 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:58:37.313 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T21:58:37.315 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:58:37.316 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_13 2026-02-22T21:58:37.316 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:3 2026-02-22T21:58:37.316 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:58:37.316 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:58:37.316 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T21:58:37.316 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:3 2026-02-22T21:58:37.316 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:58:37.316 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:58:37.316 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:58:37.316 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=3 2026-02-22T21:58:37.316 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T21:58:37.316 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:58:37.316 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:58:37.316 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=3 2026-02-22T21:58:37.317 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=3 2026-02-22T21:58:37.317 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T21:58:37.317 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:3 2026-02-22T21:58:37.317 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:58:37.317 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:58:37.317 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T21:58:37.317 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:3 2026-02-22T21:58:37.317 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:58:37.317 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:58:37.317 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:58:37.317 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=3 2026-02-22T21:58:37.317 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T21:58:37.317 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:58:37.317 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:58:37.317 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=3 2026-02-22T21:58:37.317 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=3 2026-02-22T21:58:37.317 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T21:58:37.327 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:58:37.327 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:58:37.327 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:58:37.327 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_13 2026-02-22T21:58:37.372 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_13'] 2026-02-22T21:58:37.372 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:58:37.372 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T21:58:37.373 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:58:37.373 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sleep 8 2026-02-22T21:58:45.374 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:58:45.374 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ seq 0 3 2026-02-22T21:58:45.376 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:58:45.376 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_13 2026-02-22T21:58:45.376 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:0 2026-02-22T21:58:45.376 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:58:45.376 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:58:45.376 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:58:45.376 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:0 2026-02-22T21:58:45.376 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:58:45.376 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:58:45.376 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:58:45.376 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=0 2026-02-22T21:58:45.376 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:58:45.376 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:58:45.376 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:58:45.376 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=0 2026-02-22T21:58:45.377 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T21:58:45.377 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:58:45.377 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:58:45.377 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:58:45.377 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:58:45.377 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:58:45.377 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:0 2026-02-22T21:58:45.377 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:58:45.377 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:58:45.377 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:58:45.377 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T21:58:45.377 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:58:45.378 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:58:45.378 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:58:45.378 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=0 2026-02-22T21:58:45.378 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=0 2026-02-22T21:58:45.378 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:58:45.393 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:58:45.393 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:58:45.393 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:58:45.393 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_13 2026-02-22T21:58:45.442 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ return 0 2026-02-22T21:58:45.443 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:58:45.443 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:58:45.444 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep -i Replaying 2026-02-22T21:58:45.444 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: "state": "Replaying" 2026-02-22T21:58:45.447 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_state=started 2026-02-22T21:58:45.447 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test started = started 2026-02-22T21:58:45.447 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:58:45.447 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_13 2026-02-22T21:58:45.447 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T21:58:45.448 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:58:45.448 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:58:45.448 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_13 2026-02-22T21:58:45.448 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = journal ']' 2026-02-22T21:58:45.448 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T21:58:45.448 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ mirror_image_snapshot cluster2 mirror image_13 2026-02-22T21:58:45.449 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:58:45.449 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:58:45.449 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_13 2026-02-22T21:58:45.449 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_13 2026-02-22T21:58:46.183 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Snapshot ID: 110 2026-02-22T21:58:46.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_13 2026-02-22T21:58:46.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T21:58:46.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:58:46.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:58:46.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_13 2026-02-22T21:58:46.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster2-mirror-image_13.status 2026-02-22T21:58:46.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster2-mirror-image_13.status 2026-02-22T21:58:46.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T21:58:46.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_log=/tmp/tmp.j44LZPowal/cluster2-mirror-image_13.status 2026-02-22T21:58:46.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster1-mirror-image_13.status 2026-02-22T21:58:46.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster1-mirror-image_13.status 2026-02-22T21:58:46.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T21:58:46.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_status_log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_13.status 2026-02-22T21:58:46.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_13 /tmp/tmp.j44LZPowal/cluster2-mirror-image_13.status 2026-02-22T21:58:46.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:58:46.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:58:46.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_13 2026-02-22T21:58:46.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster2-mirror-image_13.status 2026-02-22T21:58:46.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_13 --format xml 2026-02-22T21:58:46.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:58:46.220 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T21:58:46.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snapshot_id=110 2026-02-22T21:58:46.222 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:58:46.222 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:58:46.222 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.2 2026-02-22T21:58:46.422 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_13 /tmp/tmp.j44LZPowal/cluster1-mirror-image_13.status 2026-02-22T21:58:46.423 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:58:46.423 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:58:46.423 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_13 2026-02-22T21:58:46.423 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_13.status 2026-02-22T21:58:46.423 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_13 --format xml 2026-02-22T21:58:46.423 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:58:46.453 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:58:46.453 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:58:46.454 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T21:58:46.454 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T21:58:46.454 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T21:58:46.455 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T21:58:46.455 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 110 = '' 2026-02-22T21:58:46.455 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:58:46.455 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.4 2026-02-22T21:58:46.856 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_13 /tmp/tmp.j44LZPowal/cluster1-mirror-image_13.status 2026-02-22T21:58:46.857 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:58:46.857 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:58:46.857 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_13 2026-02-22T21:58:46.857 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_13.status 2026-02-22T21:58:46.857 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_13 --format xml 2026-02-22T21:58:46.857 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:58:46.886 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:58:46.886 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:58:46.887 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T21:58:46.887 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T21:58:46.887 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T21:58:46.888 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T21:58:46.888 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 110 = '' 2026-02-22T21:58:46.888 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:58:46.888 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.8 2026-02-22T21:58:47.689 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_13 /tmp/tmp.j44LZPowal/cluster1-mirror-image_13.status 2026-02-22T21:58:47.689 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:58:47.689 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:58:47.690 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_13 2026-02-22T21:58:47.690 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_13.status 2026-02-22T21:58:47.690 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_13 --format xml 2026-02-22T21:58:47.690 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:58:47.720 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:58:47.721 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id=110 2026-02-22T21:58:47.722 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 110 = 110 2026-02-22T21:58:47.722 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:58:47.722 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snap_present cluster1 mirror image_13 snap 2026-02-22T21:58:47.722 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:58:47.722 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:58:47.722 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_13 2026-02-22T21:58:47.722 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T21:58:47.722 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T21:58:47.722 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T21:58:47.722 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T21:58:48.722 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 info mirror/image_13@snap 2026-02-22T21:58:48.749 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_13': 2026-02-22T21:58:48.750 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 128 MiB in 32 objects 2026-02-22T21:58:48.750 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T21:58:48.750 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 2 2026-02-22T21:58:48.750 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 119a4f847321 2026-02-22T21:58:48.750 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.119a4f847321 2026-02-22T21:58:48.750 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T21:58:48.750 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock, non-primary 2026-02-22T21:58:48.751 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T21:58:48.751 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T21:58:48.751 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:50:02 2026 2026-02-22T21:58:48.751 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:50:02 2026 2026-02-22T21:58:48.751 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:50:02 2026 2026-02-22T21:58:48.751 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: protected: False 2026-02-22T21:58:48.751 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T21:58:48.751 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T21:58:48.751 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: 5d0b4c97-6ffd-4ecc-87a0-5cb43cf91b88 2026-02-22T21:58:48.751 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: false 2026-02-22T21:58:48.753 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:58:48.753 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ compare_image_snaps mirror image_13 snap 2026-02-22T21:58:48.753 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:58:48.753 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_13 2026-02-22T21:58:48.753 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T21:58:48.753 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local ret=0 2026-02-22T21:58:48.753 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local rmt_export=/tmp/tmp.j44LZPowal/cluster2-mirror-image_13.export 2026-02-22T21:58:48.753 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local loc_export=/tmp/tmp.j44LZPowal/cluster1-mirror-image_13.export 2026-02-22T21:58:48.753 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-image_13.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_13.export 2026-02-22T21:58:48.754 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror export image_13@snap /tmp/tmp.j44LZPowal/cluster2-mirror-image_13.export 2026-02-22T21:58:48.855 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T21:58:48.863 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 -p mirror export image_13@snap /tmp/tmp.j44LZPowal/cluster1-mirror-image_13.export 2026-02-22T21:58:48.978 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T21:58:48.985 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ cmp /tmp/tmp.j44LZPowal/cluster2-mirror-image_13.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_13.export 2026-02-22T21:58:49.033 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-image_13.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_13.export 2026-02-22T21:58:49.066 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:58:49.066 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T21:58:49.066 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_14 2026-02-22T21:58:49.066 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_snap cluster2 mirror image_14 snap 2026-02-22T21:58:49.066 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:58:49.066 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:58:49.066 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_14 2026-02-22T21:58:49.066 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T21:58:49.066 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror snap create image_14@snap --debug-rbd=20 --debug-journaler=20 2026-02-22T21:58:50.188 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_started cluster1 mirror image_14 2026-02-22T21:58:50.188 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:58:50.188 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:58:50.188 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_14 2026-02-22T21:58:50.188 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_state cluster1 mirror image_14 started 2026-02-22T21:58:50.188 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:58:50.188 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:58:50.188 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_14 2026-02-22T21:58:50.188 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=started 2026-02-22T21:58:50.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T21:58:50.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:58:50.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T21:58:51.188 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_replay_state cluster1 mirror image_14 started 2026-02-22T21:58:51.188 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:58:51.188 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:58:51.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_14 2026-02-22T21:58:51.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=started 2026-02-22T21:58:51.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_result 2026-02-22T21:58:51.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=stopped 2026-02-22T21:58:51.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_14 2026-02-22T21:58:51.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_instance=cluster1 2026-02-22T21:58:51.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:58:51.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T21:58:51.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance=cluster1 2026-02-22T21:58:51.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local loop_instance 2026-02-22T21:58:51.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ grep -i state 2026-02-22T21:58:51.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:58:51.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sleep 0 2026-02-22T21:58:51.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:58:51.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ seq 0 3 2026-02-22T21:58:51.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:58:51.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_14 2026-02-22T21:58:51.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:0 2026-02-22T21:58:51.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:58:51.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:58:51.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:58:51.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:0 2026-02-22T21:58:51.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:58:51.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:58:51.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:58:51.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=0 2026-02-22T21:58:51.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:58:51.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:58:51.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:58:51.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=0 2026-02-22T21:58:51.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T21:58:51.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:58:51.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:58:51.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:58:51.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:58:51.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:58:51.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:0 2026-02-22T21:58:51.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:58:51.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:58:51.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:58:51.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T21:58:51.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:58:51.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:58:51.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:58:51.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=0 2026-02-22T21:58:51.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=0 2026-02-22T21:58:51.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:58:51.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:58:51.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:58:51.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:58:51.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_14 2026-02-22T21:58:51.251 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_14'] 2026-02-22T21:58:51.251 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:58:51.251 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T21:58:51.254 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:58:51.254 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_14 2026-02-22T21:58:51.254 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:1 2026-02-22T21:58:51.254 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:58:51.254 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:58:51.254 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T21:58:51.254 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:1 2026-02-22T21:58:51.254 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:58:51.254 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:58:51.254 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:58:51.254 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=1 2026-02-22T21:58:51.254 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T21:58:51.255 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:58:51.255 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:58:51.255 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=1 2026-02-22T21:58:51.255 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T21:58:51.255 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T21:58:51.255 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:1 2026-02-22T21:58:51.255 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:58:51.255 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:58:51.255 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T21:58:51.255 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:1 2026-02-22T21:58:51.255 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:58:51.255 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:58:51.255 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:58:51.255 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T21:58:51.255 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T21:58:51.256 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:58:51.256 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:58:51.256 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=1 2026-02-22T21:58:51.256 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=1 2026-02-22T21:58:51.256 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T21:58:51.265 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:58:51.266 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:58:51.266 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:58:51.266 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_14 2026-02-22T21:58:51.311 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_14'] 2026-02-22T21:58:51.311 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:58:51.311 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T21:58:51.314 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:58:51.314 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_14 2026-02-22T21:58:51.314 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:2 2026-02-22T21:58:51.314 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:58:51.314 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:58:51.314 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T21:58:51.314 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:2 2026-02-22T21:58:51.315 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:58:51.315 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:58:51.315 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:58:51.315 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=2 2026-02-22T21:58:51.315 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T21:58:51.315 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:58:51.315 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:58:51.315 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=2 2026-02-22T21:58:51.315 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T21:58:51.315 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T21:58:51.315 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:2 2026-02-22T21:58:51.315 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:58:51.316 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:58:51.316 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T21:58:51.316 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:2 2026-02-22T21:58:51.316 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:58:51.316 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:58:51.316 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:58:51.316 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T21:58:51.316 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T21:58:51.316 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:58:51.316 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:58:51.316 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=2 2026-02-22T21:58:51.316 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=2 2026-02-22T21:58:51.317 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T21:58:51.324 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:58:51.324 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:58:51.325 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:58:51.325 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_14 2026-02-22T21:58:51.367 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_14'] 2026-02-22T21:58:51.368 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:58:51.368 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T21:58:51.370 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:58:51.370 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_14 2026-02-22T21:58:51.371 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:3 2026-02-22T21:58:51.371 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:58:51.371 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:58:51.371 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T21:58:51.371 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:3 2026-02-22T21:58:51.371 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:58:51.371 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:58:51.371 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:58:51.371 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=3 2026-02-22T21:58:51.371 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T21:58:51.371 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:58:51.372 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:58:51.372 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=3 2026-02-22T21:58:51.372 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=3 2026-02-22T21:58:51.372 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T21:58:51.372 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:3 2026-02-22T21:58:51.372 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:58:51.372 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:58:51.372 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T21:58:51.372 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:3 2026-02-22T21:58:51.372 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:58:51.372 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:58:51.373 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:58:51.373 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=3 2026-02-22T21:58:51.373 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T21:58:51.373 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:58:51.373 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:58:51.373 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=3 2026-02-22T21:58:51.373 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=3 2026-02-22T21:58:51.373 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T21:58:51.381 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:58:51.381 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:58:51.381 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:58:51.382 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_14 2026-02-22T21:58:51.430 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ return 0 2026-02-22T21:58:51.431 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:58:51.431 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:58:51.431 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep -i Replaying 2026-02-22T21:58:51.432 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: "state": "Replaying" 2026-02-22T21:58:51.433 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_state=started 2026-02-22T21:58:51.434 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test started = started 2026-02-22T21:58:51.434 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:58:51.434 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_14 2026-02-22T21:58:51.434 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T21:58:51.434 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:58:51.434 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:58:51.434 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_14 2026-02-22T21:58:51.434 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = journal ']' 2026-02-22T21:58:51.434 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T21:58:51.434 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ mirror_image_snapshot cluster2 mirror image_14 2026-02-22T21:58:51.435 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:58:51.435 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:58:51.435 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_14 2026-02-22T21:58:51.435 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_14 2026-02-22T21:58:52.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Snapshot ID: 112 2026-02-22T21:58:52.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_14 2026-02-22T21:58:52.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T21:58:52.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:58:52.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:58:52.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_14 2026-02-22T21:58:52.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster2-mirror-image_14.status 2026-02-22T21:58:52.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T21:58:52.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster2-mirror-image_14.status 2026-02-22T21:58:52.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_log=/tmp/tmp.j44LZPowal/cluster2-mirror-image_14.status 2026-02-22T21:58:52.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster1-mirror-image_14.status 2026-02-22T21:58:52.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster1-mirror-image_14.status 2026-02-22T21:58:52.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T21:58:52.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_status_log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_14.status 2026-02-22T21:58:52.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_14 /tmp/tmp.j44LZPowal/cluster2-mirror-image_14.status 2026-02-22T21:58:52.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:58:52.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:58:52.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_14 2026-02-22T21:58:52.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster2-mirror-image_14.status 2026-02-22T21:58:52.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_14 --format xml 2026-02-22T21:58:52.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:58:52.231 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T21:58:52.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snapshot_id=112 2026-02-22T21:58:52.233 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:58:52.233 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:58:52.233 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.2 2026-02-22T21:58:52.434 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_14 /tmp/tmp.j44LZPowal/cluster1-mirror-image_14.status 2026-02-22T21:58:52.435 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:58:52.435 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:58:52.435 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_14 2026-02-22T21:58:52.435 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_14.status 2026-02-22T21:58:52.435 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_14 --format xml 2026-02-22T21:58:52.435 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:58:52.463 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:58:52.463 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:58:52.464 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T21:58:52.464 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T21:58:52.464 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T21:58:52.464 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T21:58:52.465 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 112 = '' 2026-02-22T21:58:52.465 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:58:52.465 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.4 2026-02-22T21:58:52.866 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_14 /tmp/tmp.j44LZPowal/cluster1-mirror-image_14.status 2026-02-22T21:58:52.866 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:58:52.867 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:58:52.867 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_14 2026-02-22T21:58:52.867 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_14.status 2026-02-22T21:58:52.867 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_14 --format xml 2026-02-22T21:58:52.867 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:58:52.895 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:58:52.895 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:58:52.896 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T21:58:52.896 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T21:58:52.897 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T21:58:52.897 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T21:58:52.897 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 112 = '' 2026-02-22T21:58:52.897 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:58:52.897 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.8 2026-02-22T21:58:53.699 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_14 /tmp/tmp.j44LZPowal/cluster1-mirror-image_14.status 2026-02-22T21:58:53.699 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:58:53.699 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:58:53.699 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_14 2026-02-22T21:58:53.699 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_14.status 2026-02-22T21:58:53.699 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_14 --format xml 2026-02-22T21:58:53.700 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:58:53.729 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:58:53.731 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id=112 2026-02-22T21:58:53.731 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 112 = 112 2026-02-22T21:58:53.731 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:58:53.731 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snap_present cluster1 mirror image_14 snap 2026-02-22T21:58:53.731 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:58:53.731 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:58:53.732 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_14 2026-02-22T21:58:53.732 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T21:58:53.732 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T21:58:53.732 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T21:58:53.732 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T21:58:54.731 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 info mirror/image_14@snap 2026-02-22T21:58:54.758 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_14': 2026-02-22T21:58:54.758 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 128 MiB in 32 objects 2026-02-22T21:58:54.758 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T21:58:54.758 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 2 2026-02-22T21:58:54.758 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 11bdb52013d0 2026-02-22T21:58:54.758 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.11bdb52013d0 2026-02-22T21:58:54.759 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T21:58:54.759 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock, non-primary 2026-02-22T21:58:54.759 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T21:58:54.759 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T21:58:54.759 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:50:12 2026 2026-02-22T21:58:54.759 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:50:12 2026 2026-02-22T21:58:54.759 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:50:12 2026 2026-02-22T21:58:54.759 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: protected: False 2026-02-22T21:58:54.759 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T21:58:54.759 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T21:58:54.759 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: adaf312a-e129-41dc-a0bd-6d745384fd91 2026-02-22T21:58:54.759 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: false 2026-02-22T21:58:54.761 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:58:54.761 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ compare_image_snaps mirror image_14 snap 2026-02-22T21:58:54.761 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:58:54.761 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_14 2026-02-22T21:58:54.761 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T21:58:54.761 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local ret=0 2026-02-22T21:58:54.761 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local rmt_export=/tmp/tmp.j44LZPowal/cluster2-mirror-image_14.export 2026-02-22T21:58:54.761 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local loc_export=/tmp/tmp.j44LZPowal/cluster1-mirror-image_14.export 2026-02-22T21:58:54.761 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-image_14.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_14.export 2026-02-22T21:58:54.762 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror export image_14@snap /tmp/tmp.j44LZPowal/cluster2-mirror-image_14.export 2026-02-22T21:58:54.872 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T21:58:54.877 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 -p mirror export image_14@snap /tmp/tmp.j44LZPowal/cluster1-mirror-image_14.export 2026-02-22T21:58:55.000 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T21:58:55.009 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ cmp /tmp/tmp.j44LZPowal/cluster2-mirror-image_14.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_14.export 2026-02-22T21:58:55.058 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-image_14.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_14.export 2026-02-22T21:58:55.093 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:58:55.093 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T21:58:55.094 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_15 2026-02-22T21:58:55.094 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_snap cluster2 mirror image_15 snap 2026-02-22T21:58:55.094 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:58:55.094 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:58:55.094 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_15 2026-02-22T21:58:55.094 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T21:58:55.094 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror snap create image_15@snap --debug-rbd=20 --debug-journaler=20 2026-02-22T21:58:56.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_started cluster1 mirror image_15 2026-02-22T21:58:56.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:58:56.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:58:56.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_15 2026-02-22T21:58:56.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_state cluster1 mirror image_15 started 2026-02-22T21:58:56.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:58:56.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:58:56.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_15 2026-02-22T21:58:56.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=started 2026-02-22T21:58:56.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T21:58:56.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:58:56.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T21:58:57.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_replay_state cluster1 mirror image_15 started 2026-02-22T21:58:57.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:58:57.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:58:57.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_15 2026-02-22T21:58:57.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=started 2026-02-22T21:58:57.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_result 2026-02-22T21:58:57.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=stopped 2026-02-22T21:58:57.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_15 2026-02-22T21:58:57.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_instance=cluster1 2026-02-22T21:58:57.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:58:57.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T21:58:57.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance=cluster1 2026-02-22T21:58:57.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local loop_instance 2026-02-22T21:58:57.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:58:57.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sleep 0 2026-02-22T21:58:57.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ grep -i state 2026-02-22T21:58:57.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:58:57.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ seq 0 3 2026-02-22T21:58:57.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:58:57.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_15 2026-02-22T21:58:57.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:0 2026-02-22T21:58:57.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:58:57.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:58:57.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:58:57.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:0 2026-02-22T21:58:57.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:58:57.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:58:57.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:58:57.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=0 2026-02-22T21:58:57.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:58:57.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:58:57.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:58:57.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=0 2026-02-22T21:58:57.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T21:58:57.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:58:57.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:58:57.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:58:57.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:58:57.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:58:57.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:0 2026-02-22T21:58:57.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:58:57.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:58:57.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:58:57.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T21:58:57.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:58:57.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:58:57.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:58:57.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=0 2026-02-22T21:58:57.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=0 2026-02-22T21:58:57.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:58:57.214 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:58:57.214 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:58:57.214 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:58:57.214 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_15 2026-02-22T21:58:57.261 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_15'] 2026-02-22T21:58:57.261 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:58:57.261 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T21:58:57.265 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:58:57.265 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_15 2026-02-22T21:58:57.265 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:1 2026-02-22T21:58:57.265 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:58:57.265 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:58:57.265 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T21:58:57.266 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:1 2026-02-22T21:58:57.266 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:58:57.266 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:58:57.266 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:58:57.266 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=1 2026-02-22T21:58:57.266 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T21:58:57.266 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:58:57.266 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:58:57.266 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=1 2026-02-22T21:58:57.267 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T21:58:57.267 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T21:58:57.267 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:1 2026-02-22T21:58:57.267 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:58:57.267 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:58:57.267 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T21:58:57.267 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:1 2026-02-22T21:58:57.267 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:58:57.267 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:58:57.267 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:58:57.268 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T21:58:57.268 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T21:58:57.268 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:58:57.268 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:58:57.268 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=1 2026-02-22T21:58:57.268 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=1 2026-02-22T21:58:57.268 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T21:58:57.276 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:58:57.276 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:58:57.276 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:58:57.276 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_15 2026-02-22T21:58:57.325 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ return 0 2026-02-22T21:58:57.326 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:58:57.326 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:58:57.326 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep -i Replaying 2026-02-22T21:58:57.327 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: "state": "Replaying" 2026-02-22T21:58:57.328 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_state=started 2026-02-22T21:58:57.328 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test started = started 2026-02-22T21:58:57.328 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:58:57.328 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_15 2026-02-22T21:58:57.329 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T21:58:57.329 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:58:57.329 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:58:57.329 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_15 2026-02-22T21:58:57.329 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = journal ']' 2026-02-22T21:58:57.329 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T21:58:57.329 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ mirror_image_snapshot cluster2 mirror image_15 2026-02-22T21:58:57.329 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:58:57.329 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:58:57.329 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_15 2026-02-22T21:58:57.329 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_15 2026-02-22T21:58:58.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Snapshot ID: 114 2026-02-22T21:58:58.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_15 2026-02-22T21:58:58.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T21:58:58.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:58:58.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:58:58.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_15 2026-02-22T21:58:58.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster2-mirror-image_15.status 2026-02-22T21:58:58.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster2-mirror-image_15.status 2026-02-22T21:58:58.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T21:58:58.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_log=/tmp/tmp.j44LZPowal/cluster2-mirror-image_15.status 2026-02-22T21:58:58.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster1-mirror-image_15.status 2026-02-22T21:58:58.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster1-mirror-image_15.status 2026-02-22T21:58:58.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T21:58:58.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_status_log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_15.status 2026-02-22T21:58:58.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_15 /tmp/tmp.j44LZPowal/cluster2-mirror-image_15.status 2026-02-22T21:58:58.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:58:58.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:58:58.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_15 2026-02-22T21:58:58.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster2-mirror-image_15.status 2026-02-22T21:58:58.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_15 --format xml 2026-02-22T21:58:58.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:58:58.235 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T21:58:58.236 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snapshot_id=114 2026-02-22T21:58:58.237 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:58:58.237 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:58:58.237 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.2 2026-02-22T21:58:58.438 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_15 /tmp/tmp.j44LZPowal/cluster1-mirror-image_15.status 2026-02-22T21:58:58.438 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:58:58.438 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:58:58.438 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_15 2026-02-22T21:58:58.438 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_15.status 2026-02-22T21:58:58.439 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_15 --format xml 2026-02-22T21:58:58.439 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:58:58.467 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:58:58.467 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:58:58.468 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T21:58:58.468 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T21:58:58.468 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T21:58:58.469 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T21:58:58.469 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 114 = '' 2026-02-22T21:58:58.469 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:58:58.469 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.4 2026-02-22T21:58:58.870 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_15 /tmp/tmp.j44LZPowal/cluster1-mirror-image_15.status 2026-02-22T21:58:58.870 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:58:58.870 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:58:58.870 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_15 2026-02-22T21:58:58.871 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_15.status 2026-02-22T21:58:58.871 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_15 --format xml 2026-02-22T21:58:58.871 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:58:58.900 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:58:58.901 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:58:58.902 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T21:58:58.902 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T21:58:58.902 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T21:58:58.903 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T21:58:58.903 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 114 = '' 2026-02-22T21:58:58.903 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:58:58.903 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.8 2026-02-22T21:58:59.704 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_15 /tmp/tmp.j44LZPowal/cluster1-mirror-image_15.status 2026-02-22T21:58:59.704 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:58:59.704 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:58:59.704 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_15 2026-02-22T21:58:59.704 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_15.status 2026-02-22T21:58:59.704 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_15 --format xml 2026-02-22T21:58:59.704 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:58:59.733 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:58:59.735 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id=114 2026-02-22T21:58:59.735 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 114 = 114 2026-02-22T21:58:59.735 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:58:59.735 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snap_present cluster1 mirror image_15 snap 2026-02-22T21:58:59.736 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:58:59.736 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:58:59.736 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_15 2026-02-22T21:58:59.736 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T21:58:59.736 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T21:58:59.736 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T21:58:59.736 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T21:59:00.736 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 info mirror/image_15@snap 2026-02-22T21:59:00.763 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_15': 2026-02-22T21:59:00.763 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 128 MiB in 32 objects 2026-02-22T21:59:00.763 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T21:59:00.764 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 2 2026-02-22T21:59:00.764 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 1227af47937b 2026-02-22T21:59:00.764 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.1227af47937b 2026-02-22T21:59:00.764 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T21:59:00.764 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock, non-primary 2026-02-22T21:59:00.764 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T21:59:00.764 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T21:59:00.764 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:50:22 2026 2026-02-22T21:59:00.764 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:50:22 2026 2026-02-22T21:59:00.764 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:50:22 2026 2026-02-22T21:59:00.764 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: protected: False 2026-02-22T21:59:00.764 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T21:59:00.765 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T21:59:00.765 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: c476b3b3-1c03-4616-860c-bdf602d8dfcf 2026-02-22T21:59:00.765 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: false 2026-02-22T21:59:00.766 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:59:00.766 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ compare_image_snaps mirror image_15 snap 2026-02-22T21:59:00.766 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:00.766 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_15 2026-02-22T21:59:00.766 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T21:59:00.766 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local ret=0 2026-02-22T21:59:00.767 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local rmt_export=/tmp/tmp.j44LZPowal/cluster2-mirror-image_15.export 2026-02-22T21:59:00.767 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local loc_export=/tmp/tmp.j44LZPowal/cluster1-mirror-image_15.export 2026-02-22T21:59:00.767 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-image_15.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_15.export 2026-02-22T21:59:00.767 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror export image_15@snap /tmp/tmp.j44LZPowal/cluster2-mirror-image_15.export 2026-02-22T21:59:00.873 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T21:59:00.881 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 -p mirror export image_15@snap /tmp/tmp.j44LZPowal/cluster1-mirror-image_15.export 2026-02-22T21:59:01.009 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T21:59:01.017 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ cmp /tmp/tmp.j44LZPowal/cluster2-mirror-image_15.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_15.export 2026-02-22T21:59:01.065 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-image_15.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_15.export 2026-02-22T21:59:01.099 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:59:01.099 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T21:59:01.099 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_16 2026-02-22T21:59:01.099 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_snap cluster2 mirror image_16 snap 2026-02-22T21:59:01.100 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:59:01.100 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:01.100 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_16 2026-02-22T21:59:01.100 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T21:59:01.100 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror snap create image_16@snap --debug-rbd=20 --debug-journaler=20 2026-02-22T21:59:02.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_started cluster1 mirror image_16 2026-02-22T21:59:02.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:59:02.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:02.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_16 2026-02-22T21:59:02.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_state cluster1 mirror image_16 started 2026-02-22T21:59:02.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:59:02.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:02.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_16 2026-02-22T21:59:02.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=started 2026-02-22T21:59:02.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T21:59:02.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:59:02.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T21:59:03.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_replay_state cluster1 mirror image_16 started 2026-02-22T21:59:03.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:59:03.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:03.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_16 2026-02-22T21:59:03.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=started 2026-02-22T21:59:03.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_result 2026-02-22T21:59:03.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=stopped 2026-02-22T21:59:03.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_16 2026-02-22T21:59:03.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_instance=cluster1 2026-02-22T21:59:03.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ grep -i state 2026-02-22T21:59:03.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:59:03.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T21:59:03.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance=cluster1 2026-02-22T21:59:03.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local loop_instance 2026-02-22T21:59:03.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:59:03.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sleep 0 2026-02-22T21:59:03.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:59:03.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ seq 0 3 2026-02-22T21:59:03.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:59:03.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_16 2026-02-22T21:59:03.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:0 2026-02-22T21:59:03.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:59:03.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:59:03.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:59:03.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:0 2026-02-22T21:59:03.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:59:03.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:59:03.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:59:03.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=0 2026-02-22T21:59:03.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:59:03.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:59:03.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:59:03.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=0 2026-02-22T21:59:03.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T21:59:03.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:59:03.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:59:03.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:59:03.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:59:03.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:59:03.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:0 2026-02-22T21:59:03.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:59:03.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:59:03.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:59:03.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T21:59:03.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:59:03.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:59:03.213 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:59:03.213 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=0 2026-02-22T21:59:03.213 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=0 2026-02-22T21:59:03.213 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:59:03.220 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:59:03.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:59:03.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:59:03.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_16 2026-02-22T21:59:03.267 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_16'] 2026-02-22T21:59:03.267 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:59:03.267 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T21:59:03.270 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:59:03.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_16 2026-02-22T21:59:03.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:1 2026-02-22T21:59:03.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:59:03.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:59:03.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T21:59:03.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:1 2026-02-22T21:59:03.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:59:03.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:59:03.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:59:03.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=1 2026-02-22T21:59:03.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T21:59:03.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:59:03.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:59:03.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=1 2026-02-22T21:59:03.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T21:59:03.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T21:59:03.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:1 2026-02-22T21:59:03.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:59:03.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:59:03.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T21:59:03.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:1 2026-02-22T21:59:03.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:59:03.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:59:03.273 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:59:03.273 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T21:59:03.273 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T21:59:03.273 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:59:03.273 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:59:03.273 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=1 2026-02-22T21:59:03.273 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=1 2026-02-22T21:59:03.273 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T21:59:03.281 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:59:03.281 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:59:03.281 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:59:03.281 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_16 2026-02-22T21:59:03.327 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_16'] 2026-02-22T21:59:03.327 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:59:03.327 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T21:59:03.330 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:59:03.330 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_16 2026-02-22T21:59:03.330 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:2 2026-02-22T21:59:03.331 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:59:03.331 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:59:03.331 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T21:59:03.331 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:2 2026-02-22T21:59:03.331 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:59:03.331 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:59:03.331 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:59:03.331 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=2 2026-02-22T21:59:03.331 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T21:59:03.331 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:59:03.331 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:59:03.332 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=2 2026-02-22T21:59:03.332 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T21:59:03.332 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T21:59:03.332 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:2 2026-02-22T21:59:03.332 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:59:03.332 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:59:03.332 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T21:59:03.332 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:2 2026-02-22T21:59:03.332 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:59:03.332 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:59:03.332 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:59:03.332 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T21:59:03.332 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T21:59:03.333 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:59:03.333 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:59:03.333 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=2 2026-02-22T21:59:03.333 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=2 2026-02-22T21:59:03.333 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T21:59:03.340 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:59:03.341 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:59:03.341 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:59:03.341 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_16 2026-02-22T21:59:03.390 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ return 0 2026-02-22T21:59:03.391 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:59:03.391 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:59:03.391 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep -i Replaying 2026-02-22T21:59:03.392 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: "state": "Replaying" 2026-02-22T21:59:03.393 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_state=started 2026-02-22T21:59:03.394 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test started = started 2026-02-22T21:59:03.394 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:59:03.394 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_16 2026-02-22T21:59:03.394 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T21:59:03.394 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:59:03.394 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:03.394 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_16 2026-02-22T21:59:03.394 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = journal ']' 2026-02-22T21:59:03.394 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T21:59:03.394 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ mirror_image_snapshot cluster2 mirror image_16 2026-02-22T21:59:03.394 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:59:03.395 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:03.395 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_16 2026-02-22T21:59:03.395 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_16 2026-02-22T21:59:04.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Snapshot ID: 116 2026-02-22T21:59:04.214 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_16 2026-02-22T21:59:04.214 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T21:59:04.214 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:59:04.214 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:04.214 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_16 2026-02-22T21:59:04.215 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster2-mirror-image_16.status 2026-02-22T21:59:04.215 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster2-mirror-image_16.status 2026-02-22T21:59:04.215 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T21:59:04.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_log=/tmp/tmp.j44LZPowal/cluster2-mirror-image_16.status 2026-02-22T21:59:04.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster1-mirror-image_16.status 2026-02-22T21:59:04.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster1-mirror-image_16.status 2026-02-22T21:59:04.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T21:59:04.218 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_status_log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_16.status 2026-02-22T21:59:04.218 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_16 /tmp/tmp.j44LZPowal/cluster2-mirror-image_16.status 2026-02-22T21:59:04.218 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:59:04.219 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:04.219 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_16 2026-02-22T21:59:04.219 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster2-mirror-image_16.status 2026-02-22T21:59:04.219 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_16 --format xml 2026-02-22T21:59:04.219 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:59:04.246 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T21:59:04.248 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snapshot_id=116 2026-02-22T21:59:04.248 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:59:04.248 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:59:04.248 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.2 2026-02-22T21:59:04.449 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_16 /tmp/tmp.j44LZPowal/cluster1-mirror-image_16.status 2026-02-22T21:59:04.450 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:59:04.450 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:04.450 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_16 2026-02-22T21:59:04.450 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_16.status 2026-02-22T21:59:04.450 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_16 --format xml 2026-02-22T21:59:04.450 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:59:04.481 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:59:04.481 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:59:04.482 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T21:59:04.482 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T21:59:04.482 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T21:59:04.483 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T21:59:04.483 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 116 = '' 2026-02-22T21:59:04.483 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:59:04.484 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.4 2026-02-22T21:59:04.885 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_16 /tmp/tmp.j44LZPowal/cluster1-mirror-image_16.status 2026-02-22T21:59:04.885 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:59:04.886 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:04.886 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_16 2026-02-22T21:59:04.886 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_16.status 2026-02-22T21:59:04.886 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_16 --format xml 2026-02-22T21:59:04.886 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:59:04.916 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:59:04.916 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:59:04.917 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T21:59:04.917 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T21:59:04.917 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T21:59:04.918 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T21:59:04.919 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 116 = '' 2026-02-22T21:59:04.919 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:59:04.919 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.8 2026-02-22T21:59:05.719 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_16 /tmp/tmp.j44LZPowal/cluster1-mirror-image_16.status 2026-02-22T21:59:05.719 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:59:05.719 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:05.719 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_16 2026-02-22T21:59:05.719 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_16.status 2026-02-22T21:59:05.719 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_16 --format xml 2026-02-22T21:59:05.719 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:59:05.751 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:59:05.753 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id=116 2026-02-22T21:59:05.753 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 116 = 116 2026-02-22T21:59:05.754 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:59:05.754 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snap_present cluster1 mirror image_16 snap 2026-02-22T21:59:05.754 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:59:05.754 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:05.754 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_16 2026-02-22T21:59:05.754 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T21:59:05.754 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T21:59:05.754 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T21:59:05.754 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T21:59:06.754 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 info mirror/image_16@snap 2026-02-22T21:59:06.782 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_16': 2026-02-22T21:59:06.782 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 128 MiB in 32 objects 2026-02-22T21:59:06.783 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T21:59:06.783 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 2 2026-02-22T21:59:06.783 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 124890205f85 2026-02-22T21:59:06.783 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.124890205f85 2026-02-22T21:59:06.783 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T21:59:06.783 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock, non-primary 2026-02-22T21:59:06.784 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T21:59:06.784 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T21:59:06.784 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:50:31 2026 2026-02-22T21:59:06.784 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:50:31 2026 2026-02-22T21:59:06.784 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:50:31 2026 2026-02-22T21:59:06.784 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: protected: False 2026-02-22T21:59:06.784 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T21:59:06.784 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T21:59:06.784 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: 16696fcf-2c1b-4676-bcbc-e837ecbffb01 2026-02-22T21:59:06.784 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: false 2026-02-22T21:59:06.786 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:59:06.786 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ compare_image_snaps mirror image_16 snap 2026-02-22T21:59:06.786 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:06.786 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_16 2026-02-22T21:59:06.786 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T21:59:06.786 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local ret=0 2026-02-22T21:59:06.786 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local rmt_export=/tmp/tmp.j44LZPowal/cluster2-mirror-image_16.export 2026-02-22T21:59:06.786 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local loc_export=/tmp/tmp.j44LZPowal/cluster1-mirror-image_16.export 2026-02-22T21:59:06.786 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-image_16.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_16.export 2026-02-22T21:59:06.786 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror export image_16@snap /tmp/tmp.j44LZPowal/cluster2-mirror-image_16.export 2026-02-22T21:59:06.893 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T21:59:06.900 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 -p mirror export image_16@snap /tmp/tmp.j44LZPowal/cluster1-mirror-image_16.export 2026-02-22T21:59:07.026 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T21:59:07.033 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ cmp /tmp/tmp.j44LZPowal/cluster2-mirror-image_16.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_16.export 2026-02-22T21:59:07.083 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-image_16.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_16.export 2026-02-22T21:59:07.117 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:59:07.118 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T21:59:07.118 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_17 2026-02-22T21:59:07.118 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_snap cluster2 mirror image_17 snap 2026-02-22T21:59:07.118 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:59:07.118 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:07.118 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_17 2026-02-22T21:59:07.118 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T21:59:07.118 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror snap create image_17@snap --debug-rbd=20 --debug-journaler=20 2026-02-22T21:59:08.214 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_started cluster1 mirror image_17 2026-02-22T21:59:08.214 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:59:08.214 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:08.214 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_17 2026-02-22T21:59:08.214 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_state cluster1 mirror image_17 started 2026-02-22T21:59:08.214 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:59:08.214 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:08.214 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_17 2026-02-22T21:59:08.215 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=started 2026-02-22T21:59:08.215 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T21:59:08.215 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:59:08.215 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T21:59:09.215 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_replay_state cluster1 mirror image_17 started 2026-02-22T21:59:09.215 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:59:09.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:09.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_17 2026-02-22T21:59:09.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=started 2026-02-22T21:59:09.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_result 2026-02-22T21:59:09.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=stopped 2026-02-22T21:59:09.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_17 2026-02-22T21:59:09.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_instance=cluster1 2026-02-22T21:59:09.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:59:09.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ grep -i state 2026-02-22T21:59:09.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T21:59:09.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance=cluster1 2026-02-22T21:59:09.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local loop_instance 2026-02-22T21:59:09.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:59:09.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sleep 0 2026-02-22T21:59:09.220 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:59:09.220 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ seq 0 3 2026-02-22T21:59:09.220 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:59:09.220 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_17 2026-02-22T21:59:09.220 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:0 2026-02-22T21:59:09.220 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:59:09.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:59:09.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:59:09.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:0 2026-02-22T21:59:09.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:59:09.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:59:09.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:59:09.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=0 2026-02-22T21:59:09.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:59:09.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:59:09.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:59:09.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=0 2026-02-22T21:59:09.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T21:59:09.222 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:59:09.222 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:59:09.222 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:59:09.222 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:59:09.222 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:59:09.222 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:0 2026-02-22T21:59:09.222 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:59:09.222 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:59:09.222 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:59:09.222 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T21:59:09.222 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:59:09.222 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:59:09.222 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:59:09.223 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=0 2026-02-22T21:59:09.223 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=0 2026-02-22T21:59:09.223 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:59:09.230 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:59:09.230 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:59:09.231 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:59:09.231 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_17 2026-02-22T21:59:09.281 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ return 0 2026-02-22T21:59:09.282 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:59:09.282 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:59:09.282 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep -i Replaying 2026-02-22T21:59:09.282 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: "state": "Replaying" 2026-02-22T21:59:09.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_state=started 2026-02-22T21:59:09.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test started = started 2026-02-22T21:59:09.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:59:09.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_17 2026-02-22T21:59:09.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T21:59:09.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:59:09.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:09.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_17 2026-02-22T21:59:09.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = journal ']' 2026-02-22T21:59:09.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T21:59:09.285 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ mirror_image_snapshot cluster2 mirror image_17 2026-02-22T21:59:09.285 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:59:09.285 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:09.285 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_17 2026-02-22T21:59:09.285 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_17 2026-02-22T21:59:10.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Snapshot ID: 118 2026-02-22T21:59:10.222 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_17 2026-02-22T21:59:10.222 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T21:59:10.222 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:59:10.223 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:10.223 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_17 2026-02-22T21:59:10.223 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster2-mirror-image_17.status 2026-02-22T21:59:10.223 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster2-mirror-image_17.status 2026-02-22T21:59:10.223 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T21:59:10.224 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_log=/tmp/tmp.j44LZPowal/cluster2-mirror-image_17.status 2026-02-22T21:59:10.224 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster1-mirror-image_17.status 2026-02-22T21:59:10.224 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster1-mirror-image_17.status 2026-02-22T21:59:10.224 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T21:59:10.226 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_status_log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_17.status 2026-02-22T21:59:10.226 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_17 /tmp/tmp.j44LZPowal/cluster2-mirror-image_17.status 2026-02-22T21:59:10.226 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:59:10.226 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:10.226 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_17 2026-02-22T21:59:10.227 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster2-mirror-image_17.status 2026-02-22T21:59:10.227 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_17 --format xml 2026-02-22T21:59:10.227 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:59:10.254 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T21:59:10.256 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snapshot_id=118 2026-02-22T21:59:10.256 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:59:10.256 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:59:10.256 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.2 2026-02-22T21:59:10.457 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_17 /tmp/tmp.j44LZPowal/cluster1-mirror-image_17.status 2026-02-22T21:59:10.457 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:59:10.457 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:10.458 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_17 2026-02-22T21:59:10.458 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_17.status 2026-02-22T21:59:10.458 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_17 --format xml 2026-02-22T21:59:10.458 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:59:10.486 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:59:10.486 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:59:10.487 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T21:59:10.487 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T21:59:10.487 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T21:59:10.488 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T21:59:10.488 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 118 = '' 2026-02-22T21:59:10.488 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:59:10.489 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.4 2026-02-22T21:59:10.889 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_17 /tmp/tmp.j44LZPowal/cluster1-mirror-image_17.status 2026-02-22T21:59:10.890 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:59:10.890 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:10.890 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_17 2026-02-22T21:59:10.890 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_17.status 2026-02-22T21:59:10.890 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_17 --format xml 2026-02-22T21:59:10.890 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:59:10.921 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:59:10.921 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:59:10.922 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T21:59:10.922 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T21:59:10.923 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T21:59:10.923 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T21:59:10.923 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 118 = '' 2026-02-22T21:59:10.924 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:59:10.924 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.8 2026-02-22T21:59:11.724 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_17 /tmp/tmp.j44LZPowal/cluster1-mirror-image_17.status 2026-02-22T21:59:11.724 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:59:11.724 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:11.724 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_17 2026-02-22T21:59:11.724 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_17.status 2026-02-22T21:59:11.724 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_17 --format xml 2026-02-22T21:59:11.724 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:59:11.756 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:59:11.758 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id=118 2026-02-22T21:59:11.758 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 118 = 118 2026-02-22T21:59:11.758 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:59:11.758 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snap_present cluster1 mirror image_17 snap 2026-02-22T21:59:11.758 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:59:11.758 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:11.758 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_17 2026-02-22T21:59:11.758 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T21:59:11.759 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T21:59:11.759 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T21:59:11.759 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T21:59:12.758 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 info mirror/image_17@snap 2026-02-22T21:59:12.786 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_17': 2026-02-22T21:59:12.786 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 128 MiB in 32 objects 2026-02-22T21:59:12.786 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T21:59:12.786 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 2 2026-02-22T21:59:12.787 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 119a235e165d 2026-02-22T21:59:12.787 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.119a235e165d 2026-02-22T21:59:12.787 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T21:59:12.787 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock, non-primary 2026-02-22T21:59:12.787 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T21:59:12.787 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T21:59:12.787 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:50:43 2026 2026-02-22T21:59:12.787 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:50:43 2026 2026-02-22T21:59:12.787 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:50:43 2026 2026-02-22T21:59:12.787 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: protected: False 2026-02-22T21:59:12.788 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T21:59:12.788 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T21:59:12.788 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: 0c4cedeb-aa9b-42fc-8183-74c2a5e4fd57 2026-02-22T21:59:12.788 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: false 2026-02-22T21:59:12.789 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:59:12.789 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ compare_image_snaps mirror image_17 snap 2026-02-22T21:59:12.789 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:12.789 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_17 2026-02-22T21:59:12.790 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T21:59:12.790 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local ret=0 2026-02-22T21:59:12.790 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local rmt_export=/tmp/tmp.j44LZPowal/cluster2-mirror-image_17.export 2026-02-22T21:59:12.790 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local loc_export=/tmp/tmp.j44LZPowal/cluster1-mirror-image_17.export 2026-02-22T21:59:12.790 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-image_17.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_17.export 2026-02-22T21:59:12.790 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror export image_17@snap /tmp/tmp.j44LZPowal/cluster2-mirror-image_17.export 2026-02-22T21:59:12.895 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T21:59:12.903 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 -p mirror export image_17@snap /tmp/tmp.j44LZPowal/cluster1-mirror-image_17.export 2026-02-22T21:59:13.025 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T21:59:13.033 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ cmp /tmp/tmp.j44LZPowal/cluster2-mirror-image_17.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_17.export 2026-02-22T21:59:13.083 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-image_17.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_17.export 2026-02-22T21:59:13.117 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:59:13.117 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T21:59:13.117 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_18 2026-02-22T21:59:13.118 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_snap cluster2 mirror image_18 snap 2026-02-22T21:59:13.118 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:59:13.118 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:13.118 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_18 2026-02-22T21:59:13.118 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T21:59:13.118 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror snap create image_18@snap --debug-rbd=20 --debug-journaler=20 2026-02-22T21:59:14.213 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_started cluster1 mirror image_18 2026-02-22T21:59:14.213 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:59:14.213 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:14.213 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_18 2026-02-22T21:59:14.213 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_state cluster1 mirror image_18 started 2026-02-22T21:59:14.213 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:59:14.213 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:14.213 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_18 2026-02-22T21:59:14.213 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=started 2026-02-22T21:59:14.213 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T21:59:14.213 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:59:14.213 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T21:59:15.215 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_replay_state cluster1 mirror image_18 started 2026-02-22T21:59:15.215 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:59:15.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:15.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_18 2026-02-22T21:59:15.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=started 2026-02-22T21:59:15.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_result 2026-02-22T21:59:15.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=stopped 2026-02-22T21:59:15.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_18 2026-02-22T21:59:15.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ grep -i state 2026-02-22T21:59:15.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_instance=cluster1 2026-02-22T21:59:15.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:59:15.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T21:59:15.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance=cluster1 2026-02-22T21:59:15.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local loop_instance 2026-02-22T21:59:15.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:59:15.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sleep 0 2026-02-22T21:59:15.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:59:15.220 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ seq 0 3 2026-02-22T21:59:15.220 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:59:15.220 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_18 2026-02-22T21:59:15.220 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:0 2026-02-22T21:59:15.220 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:59:15.220 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:59:15.220 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:59:15.220 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:0 2026-02-22T21:59:15.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:59:15.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:59:15.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:59:15.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=0 2026-02-22T21:59:15.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:59:15.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:59:15.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:59:15.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=0 2026-02-22T21:59:15.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T21:59:15.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:59:15.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:59:15.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:59:15.222 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:59:15.222 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:59:15.222 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:0 2026-02-22T21:59:15.222 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:59:15.222 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:59:15.222 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:59:15.222 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T21:59:15.222 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:59:15.222 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:59:15.222 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:59:15.222 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=0 2026-02-22T21:59:15.222 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=0 2026-02-22T21:59:15.223 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:59:15.229 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:59:15.229 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:59:15.230 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:59:15.230 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_18 2026-02-22T21:59:15.273 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_18'] 2026-02-22T21:59:15.273 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:59:15.273 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T21:59:15.276 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:59:15.276 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_18 2026-02-22T21:59:15.276 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:1 2026-02-22T21:59:15.276 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:59:15.277 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:59:15.277 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T21:59:15.277 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:1 2026-02-22T21:59:15.277 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:59:15.277 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:59:15.277 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:59:15.277 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=1 2026-02-22T21:59:15.277 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T21:59:15.277 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:59:15.277 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:59:15.277 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=1 2026-02-22T21:59:15.277 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T21:59:15.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T21:59:15.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:1 2026-02-22T21:59:15.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:59:15.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:59:15.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T21:59:15.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:1 2026-02-22T21:59:15.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:59:15.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:59:15.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:59:15.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T21:59:15.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T21:59:15.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:59:15.279 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:59:15.279 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=1 2026-02-22T21:59:15.279 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=1 2026-02-22T21:59:15.279 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T21:59:15.286 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:59:15.286 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:59:15.286 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:59:15.286 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_18 2026-02-22T21:59:15.331 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_18'] 2026-02-22T21:59:15.331 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:59:15.332 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T21:59:15.336 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:59:15.336 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_18 2026-02-22T21:59:15.337 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:2 2026-02-22T21:59:15.337 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:59:15.337 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:59:15.337 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T21:59:15.337 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:2 2026-02-22T21:59:15.338 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:59:15.338 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:59:15.338 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:59:15.338 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=2 2026-02-22T21:59:15.338 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T21:59:15.338 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:59:15.339 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:59:15.339 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=2 2026-02-22T21:59:15.339 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T21:59:15.339 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T21:59:15.339 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:2 2026-02-22T21:59:15.339 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:59:15.340 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:59:15.340 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T21:59:15.340 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:2 2026-02-22T21:59:15.340 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:59:15.340 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:59:15.340 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:59:15.341 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T21:59:15.341 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T21:59:15.341 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:59:15.341 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:59:15.341 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=2 2026-02-22T21:59:15.341 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=2 2026-02-22T21:59:15.341 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T21:59:15.346 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:59:15.346 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:59:15.346 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:59:15.346 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_18 2026-02-22T21:59:15.391 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_18'] 2026-02-22T21:59:15.391 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:59:15.391 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T21:59:15.394 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:59:15.394 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_18 2026-02-22T21:59:15.394 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:3 2026-02-22T21:59:15.395 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:59:15.395 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:59:15.395 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T21:59:15.395 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:3 2026-02-22T21:59:15.395 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:59:15.395 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:59:15.395 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:59:15.395 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=3 2026-02-22T21:59:15.395 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T21:59:15.395 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:59:15.395 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:59:15.395 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=3 2026-02-22T21:59:15.396 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=3 2026-02-22T21:59:15.396 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T21:59:15.396 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:3 2026-02-22T21:59:15.396 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:59:15.396 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:59:15.396 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T21:59:15.396 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:3 2026-02-22T21:59:15.396 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:59:15.396 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:59:15.396 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:59:15.396 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=3 2026-02-22T21:59:15.396 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T21:59:15.397 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:59:15.397 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:59:15.397 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=3 2026-02-22T21:59:15.397 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=3 2026-02-22T21:59:15.397 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T21:59:15.404 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:59:15.404 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:59:15.404 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:59:15.404 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_18 2026-02-22T21:59:15.454 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ return 0 2026-02-22T21:59:15.454 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:59:15.455 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:59:15.455 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep -i Replaying 2026-02-22T21:59:15.456 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: "state": "Replaying" 2026-02-22T21:59:15.457 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_state=started 2026-02-22T21:59:15.457 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test started = started 2026-02-22T21:59:15.457 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:59:15.457 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_18 2026-02-22T21:59:15.457 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T21:59:15.457 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:59:15.458 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:15.458 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_18 2026-02-22T21:59:15.458 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = journal ']' 2026-02-22T21:59:15.458 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T21:59:15.458 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ mirror_image_snapshot cluster2 mirror image_18 2026-02-22T21:59:15.458 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:59:15.458 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:15.458 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_18 2026-02-22T21:59:15.458 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_18 2026-02-22T21:59:16.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Snapshot ID: 120 2026-02-22T21:59:16.223 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_18 2026-02-22T21:59:16.223 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T21:59:16.223 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:59:16.223 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:16.223 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_18 2026-02-22T21:59:16.223 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster2-mirror-image_18.status 2026-02-22T21:59:16.224 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster2-mirror-image_18.status 2026-02-22T21:59:16.224 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T21:59:16.225 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_log=/tmp/tmp.j44LZPowal/cluster2-mirror-image_18.status 2026-02-22T21:59:16.225 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster1-mirror-image_18.status 2026-02-22T21:59:16.226 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster1-mirror-image_18.status 2026-02-22T21:59:16.226 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T21:59:16.227 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_status_log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_18.status 2026-02-22T21:59:16.227 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_18 /tmp/tmp.j44LZPowal/cluster2-mirror-image_18.status 2026-02-22T21:59:16.227 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:59:16.227 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:16.227 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_18 2026-02-22T21:59:16.227 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster2-mirror-image_18.status 2026-02-22T21:59:16.227 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_18 --format xml 2026-02-22T21:59:16.227 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:59:16.255 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T21:59:16.256 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snapshot_id=120 2026-02-22T21:59:16.256 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:59:16.256 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:59:16.256 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.2 2026-02-22T21:59:16.457 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_18 /tmp/tmp.j44LZPowal/cluster1-mirror-image_18.status 2026-02-22T21:59:16.458 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:59:16.458 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:16.458 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_18 2026-02-22T21:59:16.458 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_18.status 2026-02-22T21:59:16.458 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_18 --format xml 2026-02-22T21:59:16.458 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:59:16.489 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:59:16.490 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id=52 2026-02-22T21:59:16.491 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 120 = 52 2026-02-22T21:59:16.491 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:59:16.491 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.4 2026-02-22T21:59:16.892 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_18 /tmp/tmp.j44LZPowal/cluster1-mirror-image_18.status 2026-02-22T21:59:16.892 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:59:16.892 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:16.892 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_18 2026-02-22T21:59:16.892 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_18.status 2026-02-22T21:59:16.892 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_18 --format xml 2026-02-22T21:59:16.893 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:59:16.921 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:59:16.921 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:59:16.922 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T21:59:16.922 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T21:59:16.922 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T21:59:16.923 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T21:59:16.923 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 120 = '' 2026-02-22T21:59:16.923 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:59:16.923 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.8 2026-02-22T21:59:17.724 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_18 /tmp/tmp.j44LZPowal/cluster1-mirror-image_18.status 2026-02-22T21:59:17.724 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:59:17.725 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:17.725 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_18 2026-02-22T21:59:17.725 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_18.status 2026-02-22T21:59:17.725 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_18 --format xml 2026-02-22T21:59:17.725 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:59:17.756 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:59:17.758 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id=120 2026-02-22T21:59:17.758 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 120 = 120 2026-02-22T21:59:17.758 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:59:17.758 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snap_present cluster1 mirror image_18 snap 2026-02-22T21:59:17.758 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:59:17.759 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:17.759 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_18 2026-02-22T21:59:17.759 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T21:59:17.759 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T21:59:17.759 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T21:59:17.759 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T21:59:18.759 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 info mirror/image_18@snap 2026-02-22T21:59:18.787 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_18': 2026-02-22T21:59:18.787 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 128 MiB in 32 objects 2026-02-22T21:59:18.787 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T21:59:18.787 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 2 2026-02-22T21:59:18.787 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 11bd49523661 2026-02-22T21:59:18.787 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.11bd49523661 2026-02-22T21:59:18.788 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T21:59:18.788 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock, non-primary 2026-02-22T21:59:18.788 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T21:59:18.788 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T21:59:18.788 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:50:54 2026 2026-02-22T21:59:18.788 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:50:54 2026 2026-02-22T21:59:18.788 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:50:54 2026 2026-02-22T21:59:18.788 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: protected: False 2026-02-22T21:59:18.788 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T21:59:18.788 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T21:59:18.788 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: 4baf6590-1204-4386-a935-9b243b94dcfc 2026-02-22T21:59:18.789 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: false 2026-02-22T21:59:18.790 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:59:18.790 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ compare_image_snaps mirror image_18 snap 2026-02-22T21:59:18.790 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:18.790 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_18 2026-02-22T21:59:18.790 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T21:59:18.790 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local ret=0 2026-02-22T21:59:18.790 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local rmt_export=/tmp/tmp.j44LZPowal/cluster2-mirror-image_18.export 2026-02-22T21:59:18.790 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local loc_export=/tmp/tmp.j44LZPowal/cluster1-mirror-image_18.export 2026-02-22T21:59:18.791 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-image_18.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_18.export 2026-02-22T21:59:18.791 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror export image_18@snap /tmp/tmp.j44LZPowal/cluster2-mirror-image_18.export 2026-02-22T21:59:18.894 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T21:59:18.902 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 -p mirror export image_18@snap /tmp/tmp.j44LZPowal/cluster1-mirror-image_18.export 2026-02-22T21:59:19.028 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T21:59:19.036 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ cmp /tmp/tmp.j44LZPowal/cluster2-mirror-image_18.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_18.export 2026-02-22T21:59:19.084 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-image_18.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_18.export 2026-02-22T21:59:19.118 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:59:19.118 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T21:59:19.119 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_19 2026-02-22T21:59:19.119 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_snap cluster2 mirror image_19 snap 2026-02-22T21:59:19.119 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:59:19.119 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:19.119 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_19 2026-02-22T21:59:19.119 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T21:59:19.119 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror snap create image_19@snap --debug-rbd=20 --debug-journaler=20 2026-02-22T21:59:20.213 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_started cluster1 mirror image_19 2026-02-22T21:59:20.213 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:59:20.213 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:20.213 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_19 2026-02-22T21:59:20.213 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_state cluster1 mirror image_19 started 2026-02-22T21:59:20.213 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:59:20.213 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:20.213 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_19 2026-02-22T21:59:20.214 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=started 2026-02-22T21:59:20.214 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T21:59:20.214 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:59:20.214 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T21:59:21.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_replay_state cluster1 mirror image_19 started 2026-02-22T21:59:21.222 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:59:21.222 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:21.222 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_19 2026-02-22T21:59:21.222 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=started 2026-02-22T21:59:21.223 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_result 2026-02-22T21:59:21.223 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=stopped 2026-02-22T21:59:21.223 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_19 2026-02-22T21:59:21.223 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_instance=cluster1 2026-02-22T21:59:21.223 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:59:21.223 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T21:59:21.224 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance=cluster1 2026-02-22T21:59:21.224 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local loop_instance 2026-02-22T21:59:21.224 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:59:21.224 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sleep 0 2026-02-22T21:59:21.224 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ grep -i state 2026-02-22T21:59:21.225 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:59:21.225 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ seq 0 3 2026-02-22T21:59:21.225 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:59:21.225 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_19 2026-02-22T21:59:21.225 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:0 2026-02-22T21:59:21.225 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:59:21.226 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:59:21.226 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:59:21.226 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:0 2026-02-22T21:59:21.226 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:59:21.226 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:59:21.227 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:59:21.227 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=0 2026-02-22T21:59:21.227 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:59:21.227 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:59:21.227 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:59:21.227 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=0 2026-02-22T21:59:21.228 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T21:59:21.228 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:59:21.228 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:59:21.228 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:59:21.228 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:59:21.228 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:59:21.229 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:0 2026-02-22T21:59:21.229 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:59:21.229 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:59:21.229 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:59:21.229 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T21:59:21.230 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:59:21.230 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:59:21.230 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:59:21.230 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=0 2026-02-22T21:59:21.230 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=0 2026-02-22T21:59:21.230 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:59:21.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:59:21.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:59:21.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:59:21.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_19 2026-02-22T21:59:21.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ return 0 2026-02-22T21:59:21.279 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:59:21.279 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:59:21.279 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep -i Replaying 2026-02-22T21:59:21.280 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: "state": "Replaying" 2026-02-22T21:59:21.281 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_state=started 2026-02-22T21:59:21.281 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test started = started 2026-02-22T21:59:21.281 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:59:21.281 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_19 2026-02-22T21:59:21.281 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T21:59:21.281 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:59:21.281 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:21.281 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_19 2026-02-22T21:59:21.281 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = journal ']' 2026-02-22T21:59:21.282 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T21:59:21.282 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ mirror_image_snapshot cluster2 mirror image_19 2026-02-22T21:59:21.282 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:59:21.282 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:21.282 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_19 2026-02-22T21:59:21.282 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_19 2026-02-22T21:59:22.220 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Snapshot ID: 122 2026-02-22T21:59:22.227 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_19 2026-02-22T21:59:22.227 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T21:59:22.227 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:59:22.227 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:22.227 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_19 2026-02-22T21:59:22.228 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster2-mirror-image_19.status 2026-02-22T21:59:22.228 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster2-mirror-image_19.status 2026-02-22T21:59:22.228 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T21:59:22.229 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_log=/tmp/tmp.j44LZPowal/cluster2-mirror-image_19.status 2026-02-22T21:59:22.229 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster1-mirror-image_19.status 2026-02-22T21:59:22.230 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster1-mirror-image_19.status 2026-02-22T21:59:22.230 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T21:59:22.231 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_status_log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_19.status 2026-02-22T21:59:22.231 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_19 /tmp/tmp.j44LZPowal/cluster2-mirror-image_19.status 2026-02-22T21:59:22.231 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:59:22.231 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:22.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_19 2026-02-22T21:59:22.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster2-mirror-image_19.status 2026-02-22T21:59:22.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_19 --format xml 2026-02-22T21:59:22.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:59:22.258 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T21:59:22.260 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snapshot_id=122 2026-02-22T21:59:22.260 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:59:22.260 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:59:22.260 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.2 2026-02-22T21:59:22.461 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_19 /tmp/tmp.j44LZPowal/cluster1-mirror-image_19.status 2026-02-22T21:59:22.462 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:59:22.462 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:22.462 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_19 2026-02-22T21:59:22.462 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_19.status 2026-02-22T21:59:22.462 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_19 --format xml 2026-02-22T21:59:22.462 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:59:22.492 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:59:22.493 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id=53 2026-02-22T21:59:22.493 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 122 = 53 2026-02-22T21:59:22.493 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:59:22.493 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.4 2026-02-22T21:59:22.894 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_19 /tmp/tmp.j44LZPowal/cluster1-mirror-image_19.status 2026-02-22T21:59:22.895 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:59:22.895 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:22.895 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_19 2026-02-22T21:59:22.895 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_19.status 2026-02-22T21:59:22.895 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_19 --format xml 2026-02-22T21:59:22.895 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:59:22.927 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:59:22.928 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:59:22.929 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T21:59:22.929 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T21:59:22.930 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T21:59:22.930 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T21:59:22.930 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 122 = '' 2026-02-22T21:59:22.930 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:59:22.930 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.8 2026-02-22T21:59:23.730 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_19 /tmp/tmp.j44LZPowal/cluster1-mirror-image_19.status 2026-02-22T21:59:23.730 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:59:23.730 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:23.730 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_19 2026-02-22T21:59:23.730 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_19.status 2026-02-22T21:59:23.731 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_19 --format xml 2026-02-22T21:59:23.731 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:59:23.761 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:59:23.763 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id=122 2026-02-22T21:59:23.763 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 122 = 122 2026-02-22T21:59:23.763 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:59:23.763 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snap_present cluster1 mirror image_19 snap 2026-02-22T21:59:23.763 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:59:23.764 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:23.764 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_19 2026-02-22T21:59:23.764 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T21:59:23.764 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T21:59:23.764 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T21:59:23.764 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T21:59:24.764 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 info mirror/image_19@snap 2026-02-22T21:59:24.791 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_19': 2026-02-22T21:59:24.792 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 128 MiB in 32 objects 2026-02-22T21:59:24.792 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T21:59:24.792 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 2 2026-02-22T21:59:24.792 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 122713927700 2026-02-22T21:59:24.792 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.122713927700 2026-02-22T21:59:24.792 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T21:59:24.792 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock, non-primary 2026-02-22T21:59:24.792 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T21:59:24.792 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T21:59:24.793 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:51:07 2026 2026-02-22T21:59:24.793 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:51:07 2026 2026-02-22T21:59:24.793 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:51:07 2026 2026-02-22T21:59:24.793 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: protected: False 2026-02-22T21:59:24.793 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T21:59:24.793 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T21:59:24.793 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: 7a33a362-8595-45cc-9e4c-7b4215120e99 2026-02-22T21:59:24.793 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: false 2026-02-22T21:59:24.794 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:59:24.795 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ compare_image_snaps mirror image_19 snap 2026-02-22T21:59:24.795 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:24.795 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_19 2026-02-22T21:59:24.795 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T21:59:24.795 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local ret=0 2026-02-22T21:59:24.795 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local rmt_export=/tmp/tmp.j44LZPowal/cluster2-mirror-image_19.export 2026-02-22T21:59:24.795 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local loc_export=/tmp/tmp.j44LZPowal/cluster1-mirror-image_19.export 2026-02-22T21:59:24.795 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-image_19.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_19.export 2026-02-22T21:59:24.795 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror export image_19@snap /tmp/tmp.j44LZPowal/cluster2-mirror-image_19.export 2026-02-22T21:59:24.907 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T21:59:24.915 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 -p mirror export image_19@snap /tmp/tmp.j44LZPowal/cluster1-mirror-image_19.export 2026-02-22T21:59:25.041 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T21:59:25.049 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ cmp /tmp/tmp.j44LZPowal/cluster2-mirror-image_19.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_19.export 2026-02-22T21:59:25.098 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-image_19.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_19.export 2026-02-22T21:59:25.132 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:59:25.133 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T21:59:25.133 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_20 2026-02-22T21:59:25.133 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_snap cluster2 mirror image_20 snap 2026-02-22T21:59:25.133 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:59:25.133 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:25.133 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_20 2026-02-22T21:59:25.133 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T21:59:25.133 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror snap create image_20@snap --debug-rbd=20 --debug-journaler=20 2026-02-22T21:59:26.229 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_started cluster1 mirror image_20 2026-02-22T21:59:26.230 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:59:26.230 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:26.230 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_20 2026-02-22T21:59:26.230 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_state cluster1 mirror image_20 started 2026-02-22T21:59:26.230 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:59:26.230 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:26.230 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_20 2026-02-22T21:59:26.230 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=started 2026-02-22T21:59:26.230 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T21:59:26.230 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:59:26.230 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T21:59:27.231 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_replay_state cluster1 mirror image_20 started 2026-02-22T21:59:27.231 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:59:27.231 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:27.231 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_20 2026-02-22T21:59:27.231 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=started 2026-02-22T21:59:27.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_result 2026-02-22T21:59:27.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=stopped 2026-02-22T21:59:27.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_20 2026-02-22T21:59:27.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_instance=cluster1 2026-02-22T21:59:27.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ grep -i state 2026-02-22T21:59:27.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:59:27.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T21:59:27.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance=cluster1 2026-02-22T21:59:27.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local loop_instance 2026-02-22T21:59:27.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:59:27.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sleep 0 2026-02-22T21:59:27.235 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:59:27.236 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ seq 0 3 2026-02-22T21:59:27.236 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:59:27.236 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_20 2026-02-22T21:59:27.236 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:0 2026-02-22T21:59:27.236 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:59:27.236 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:59:27.236 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:59:27.236 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:0 2026-02-22T21:59:27.236 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:59:27.236 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:59:27.236 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:59:27.237 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=0 2026-02-22T21:59:27.237 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:59:27.237 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:59:27.237 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:59:27.237 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=0 2026-02-22T21:59:27.237 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T21:59:27.237 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:59:27.237 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:59:27.237 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:59:27.237 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:59:27.237 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:59:27.237 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:0 2026-02-22T21:59:27.237 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:59:27.238 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:59:27.238 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:59:27.238 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T21:59:27.238 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:59:27.238 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:59:27.238 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:59:27.238 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=0 2026-02-22T21:59:27.238 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=0 2026-02-22T21:59:27.238 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:59:27.246 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:59:27.247 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:59:27.247 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:59:27.247 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_20 2026-02-22T21:59:27.293 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_20'] 2026-02-22T21:59:27.293 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:59:27.293 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T21:59:27.296 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:59:27.296 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_20 2026-02-22T21:59:27.296 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:1 2026-02-22T21:59:27.296 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:59:27.296 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:59:27.296 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T21:59:27.296 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:1 2026-02-22T21:59:27.297 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:59:27.297 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:59:27.297 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:59:27.297 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=1 2026-02-22T21:59:27.297 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T21:59:27.297 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:59:27.297 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:59:27.297 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=1 2026-02-22T21:59:27.297 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T21:59:27.297 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T21:59:27.297 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:1 2026-02-22T21:59:27.297 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:59:27.298 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:59:27.298 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T21:59:27.298 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:1 2026-02-22T21:59:27.298 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:59:27.298 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:59:27.298 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:59:27.298 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T21:59:27.298 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T21:59:27.298 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:59:27.298 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:59:27.298 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=1 2026-02-22T21:59:27.298 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=1 2026-02-22T21:59:27.299 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T21:59:27.306 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:59:27.306 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:59:27.307 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:59:27.307 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_20 2026-02-22T21:59:27.352 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_20'] 2026-02-22T21:59:27.352 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:59:27.352 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T21:59:27.355 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:59:27.355 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_20 2026-02-22T21:59:27.355 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:2 2026-02-22T21:59:27.355 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:59:27.355 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:59:27.356 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T21:59:27.356 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:2 2026-02-22T21:59:27.356 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:59:27.356 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:59:27.356 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:59:27.356 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=2 2026-02-22T21:59:27.356 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T21:59:27.356 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:59:27.356 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:59:27.356 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=2 2026-02-22T21:59:27.356 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T21:59:27.356 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T21:59:27.357 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:2 2026-02-22T21:59:27.357 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:59:27.357 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:59:27.357 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T21:59:27.357 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:2 2026-02-22T21:59:27.357 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:59:27.357 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:59:27.357 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:59:27.357 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T21:59:27.357 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T21:59:27.357 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:59:27.357 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:59:27.358 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=2 2026-02-22T21:59:27.358 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=2 2026-02-22T21:59:27.358 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T21:59:27.365 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:59:27.366 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:59:27.366 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:59:27.366 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_20 2026-02-22T21:59:27.414 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ return 0 2026-02-22T21:59:27.415 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:59:27.415 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:59:27.415 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep -i Replaying 2026-02-22T21:59:27.416 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: "state": "Replaying" 2026-02-22T21:59:27.417 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_state=started 2026-02-22T21:59:27.417 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test started = started 2026-02-22T21:59:27.417 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:59:27.417 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_20 2026-02-22T21:59:27.417 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T21:59:27.417 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:59:27.417 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:27.417 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_20 2026-02-22T21:59:27.417 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = journal ']' 2026-02-22T21:59:27.417 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T21:59:27.417 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ mirror_image_snapshot cluster2 mirror image_20 2026-02-22T21:59:27.417 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:59:27.417 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:27.417 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_20 2026-02-22T21:59:27.417 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_20 2026-02-22T21:59:28.233 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Snapshot ID: 124 2026-02-22T21:59:28.240 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_20 2026-02-22T21:59:28.240 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T21:59:28.240 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:59:28.240 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:28.240 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_20 2026-02-22T21:59:28.240 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster2-mirror-image_20.status 2026-02-22T21:59:28.241 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster2-mirror-image_20.status 2026-02-22T21:59:28.241 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T21:59:28.241 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_log=/tmp/tmp.j44LZPowal/cluster2-mirror-image_20.status 2026-02-22T21:59:28.242 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster1-mirror-image_20.status 2026-02-22T21:59:28.242 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster1-mirror-image_20.status 2026-02-22T21:59:28.242 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T21:59:28.244 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_status_log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_20.status 2026-02-22T21:59:28.244 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_20 /tmp/tmp.j44LZPowal/cluster2-mirror-image_20.status 2026-02-22T21:59:28.244 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:59:28.244 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:28.244 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_20 2026-02-22T21:59:28.244 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster2-mirror-image_20.status 2026-02-22T21:59:28.244 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_20 --format xml 2026-02-22T21:59:28.244 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:59:28.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T21:59:28.273 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snapshot_id=124 2026-02-22T21:59:28.273 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:59:28.273 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:59:28.273 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.2 2026-02-22T21:59:28.474 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_20 /tmp/tmp.j44LZPowal/cluster1-mirror-image_20.status 2026-02-22T21:59:28.474 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:59:28.474 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:28.474 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_20 2026-02-22T21:59:28.474 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_20.status 2026-02-22T21:59:28.475 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_20 --format xml 2026-02-22T21:59:28.475 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:59:28.503 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:59:28.505 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id=54 2026-02-22T21:59:28.505 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 124 = 54 2026-02-22T21:59:28.505 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:59:28.505 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.4 2026-02-22T21:59:28.907 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_20 /tmp/tmp.j44LZPowal/cluster1-mirror-image_20.status 2026-02-22T21:59:28.907 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:59:28.907 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:28.907 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_20 2026-02-22T21:59:28.907 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_20.status 2026-02-22T21:59:28.907 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_20 --format xml 2026-02-22T21:59:28.907 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:59:28.937 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:59:28.937 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:59:28.938 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T21:59:28.938 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T21:59:28.938 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T21:59:28.939 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T21:59:28.939 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 124 = '' 2026-02-22T21:59:28.939 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:59:28.940 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.8 2026-02-22T21:59:29.740 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_20 /tmp/tmp.j44LZPowal/cluster1-mirror-image_20.status 2026-02-22T21:59:29.740 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:59:29.740 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:29.741 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_20 2026-02-22T21:59:29.741 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_20.status 2026-02-22T21:59:29.741 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_20 --format xml 2026-02-22T21:59:29.741 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:59:29.770 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:59:29.772 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id=124 2026-02-22T21:59:29.772 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 124 = 124 2026-02-22T21:59:29.772 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:59:29.772 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snap_present cluster1 mirror image_20 snap 2026-02-22T21:59:29.772 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:59:29.772 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:29.772 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_20 2026-02-22T21:59:29.773 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T21:59:29.773 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T21:59:29.773 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T21:59:29.773 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T21:59:30.773 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 info mirror/image_20@snap 2026-02-22T21:59:30.800 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_20': 2026-02-22T21:59:30.800 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 128 MiB in 32 objects 2026-02-22T21:59:30.800 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T21:59:30.800 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 2 2026-02-22T21:59:30.800 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 124870b771ce 2026-02-22T21:59:30.801 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.124870b771ce 2026-02-22T21:59:30.801 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T21:59:30.801 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock, non-primary 2026-02-22T21:59:30.801 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T21:59:30.801 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T21:59:30.801 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:51:18 2026 2026-02-22T21:59:30.801 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:51:18 2026 2026-02-22T21:59:30.801 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:51:18 2026 2026-02-22T21:59:30.801 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: protected: False 2026-02-22T21:59:30.801 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T21:59:30.802 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T21:59:30.802 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: 50d0516c-a4f2-4b0b-9b37-15ee9b73d3ad 2026-02-22T21:59:30.802 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: false 2026-02-22T21:59:30.803 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:59:30.803 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ compare_image_snaps mirror image_20 snap 2026-02-22T21:59:30.803 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:30.803 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_20 2026-02-22T21:59:30.803 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T21:59:30.804 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local ret=0 2026-02-22T21:59:30.804 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local rmt_export=/tmp/tmp.j44LZPowal/cluster2-mirror-image_20.export 2026-02-22T21:59:30.804 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local loc_export=/tmp/tmp.j44LZPowal/cluster1-mirror-image_20.export 2026-02-22T21:59:30.804 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-image_20.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_20.export 2026-02-22T21:59:30.804 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror export image_20@snap /tmp/tmp.j44LZPowal/cluster2-mirror-image_20.export 2026-02-22T21:59:30.907 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T21:59:30.915 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 -p mirror export image_20@snap /tmp/tmp.j44LZPowal/cluster1-mirror-image_20.export 2026-02-22T21:59:31.044 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T21:59:31.052 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ cmp /tmp/tmp.j44LZPowal/cluster2-mirror-image_20.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_20.export 2026-02-22T21:59:31.102 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-image_20.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_20.export 2026-02-22T21:59:31.138 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:59:31.138 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T21:59:31.138 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_21 2026-02-22T21:59:31.138 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_snap cluster2 mirror image_21 snap 2026-02-22T21:59:31.138 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:59:31.138 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:31.138 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_21 2026-02-22T21:59:31.139 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T21:59:31.139 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror snap create image_21@snap --debug-rbd=20 --debug-journaler=20 2026-02-22T21:59:32.236 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_started cluster1 mirror image_21 2026-02-22T21:59:32.236 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:59:32.236 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:32.236 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_21 2026-02-22T21:59:32.236 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_state cluster1 mirror image_21 started 2026-02-22T21:59:32.236 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:59:32.236 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:32.236 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_21 2026-02-22T21:59:32.236 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=started 2026-02-22T21:59:32.236 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T21:59:32.236 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:59:32.237 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T21:59:33.238 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_replay_state cluster1 mirror image_21 started 2026-02-22T21:59:33.238 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:59:33.239 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:33.239 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_21 2026-02-22T21:59:33.239 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=started 2026-02-22T21:59:33.239 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_result 2026-02-22T21:59:33.239 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=stopped 2026-02-22T21:59:33.239 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_21 2026-02-22T21:59:33.239 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_instance=cluster1 2026-02-22T21:59:33.239 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:59:33.239 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T21:59:33.239 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance=cluster1 2026-02-22T21:59:33.240 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local loop_instance 2026-02-22T21:59:33.240 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ grep -i state 2026-02-22T21:59:33.240 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:59:33.240 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sleep 0 2026-02-22T21:59:33.243 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:59:33.244 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ seq 0 3 2026-02-22T21:59:33.244 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:59:33.244 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_21 2026-02-22T21:59:33.244 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:0 2026-02-22T21:59:33.244 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:59:33.244 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:59:33.244 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:59:33.244 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:0 2026-02-22T21:59:33.244 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:59:33.244 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:59:33.244 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:59:33.245 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=0 2026-02-22T21:59:33.245 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:59:33.245 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:59:33.245 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:59:33.245 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=0 2026-02-22T21:59:33.245 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T21:59:33.245 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:59:33.245 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:59:33.245 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:59:33.245 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:59:33.245 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:59:33.245 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:0 2026-02-22T21:59:33.245 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:59:33.246 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:59:33.246 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:59:33.246 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T21:59:33.246 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:59:33.246 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:59:33.246 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:59:33.246 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=0 2026-02-22T21:59:33.246 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=0 2026-02-22T21:59:33.246 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:59:33.254 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:59:33.254 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:59:33.254 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:59:33.254 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_21 2026-02-22T21:59:33.301 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_21'] 2026-02-22T21:59:33.301 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:59:33.301 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T21:59:33.304 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:59:33.304 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_21 2026-02-22T21:59:33.304 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:1 2026-02-22T21:59:33.304 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:59:33.304 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:59:33.304 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T21:59:33.304 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:1 2026-02-22T21:59:33.304 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:59:33.304 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:59:33.304 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:59:33.304 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=1 2026-02-22T21:59:33.305 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T21:59:33.305 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:59:33.305 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:59:33.305 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=1 2026-02-22T21:59:33.305 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T21:59:33.305 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T21:59:33.305 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:1 2026-02-22T21:59:33.305 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:59:33.305 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:59:33.305 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T21:59:33.305 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:1 2026-02-22T21:59:33.305 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:59:33.305 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:59:33.305 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:59:33.305 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T21:59:33.305 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T21:59:33.306 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:59:33.306 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:59:33.306 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=1 2026-02-22T21:59:33.306 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=1 2026-02-22T21:59:33.306 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T21:59:33.316 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:59:33.316 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:59:33.316 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:59:33.316 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_21 2026-02-22T21:59:33.360 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_21'] 2026-02-22T21:59:33.361 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:59:33.361 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T21:59:33.364 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:59:33.364 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_21 2026-02-22T21:59:33.364 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:2 2026-02-22T21:59:33.364 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:59:33.364 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:59:33.364 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T21:59:33.364 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:2 2026-02-22T21:59:33.364 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:59:33.364 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:59:33.364 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:59:33.364 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=2 2026-02-22T21:59:33.365 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T21:59:33.365 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:59:33.365 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:59:33.365 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=2 2026-02-22T21:59:33.365 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T21:59:33.365 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T21:59:33.365 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:2 2026-02-22T21:59:33.365 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:59:33.365 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:59:33.365 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T21:59:33.365 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:2 2026-02-22T21:59:33.365 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:59:33.366 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:59:33.366 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:59:33.366 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T21:59:33.366 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T21:59:33.366 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:59:33.366 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:59:33.366 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=2 2026-02-22T21:59:33.366 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=2 2026-02-22T21:59:33.366 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T21:59:33.374 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:59:33.374 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:59:33.374 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:59:33.374 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_21 2026-02-22T21:59:33.418 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_21'] 2026-02-22T21:59:33.418 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:59:33.418 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T21:59:33.421 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:59:33.421 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_21 2026-02-22T21:59:33.421 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:3 2026-02-22T21:59:33.421 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:59:33.421 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:59:33.421 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T21:59:33.421 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:3 2026-02-22T21:59:33.421 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:59:33.421 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:59:33.421 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:59:33.422 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=3 2026-02-22T21:59:33.422 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T21:59:33.422 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:59:33.422 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:59:33.422 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=3 2026-02-22T21:59:33.422 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=3 2026-02-22T21:59:33.422 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T21:59:33.422 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:3 2026-02-22T21:59:33.422 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:59:33.422 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:59:33.422 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T21:59:33.422 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:3 2026-02-22T21:59:33.422 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:59:33.423 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:59:33.423 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:59:33.423 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=3 2026-02-22T21:59:33.423 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T21:59:33.423 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:59:33.423 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:59:33.423 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=3 2026-02-22T21:59:33.423 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=3 2026-02-22T21:59:33.423 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T21:59:33.431 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:59:33.431 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:59:33.431 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:59:33.432 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_21 2026-02-22T21:59:33.480 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ return 0 2026-02-22T21:59:33.481 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:59:33.481 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:59:33.482 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep -i Replaying 2026-02-22T21:59:33.482 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: "state": "Replaying" 2026-02-22T21:59:33.483 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_state=started 2026-02-22T21:59:33.484 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test started = started 2026-02-22T21:59:33.484 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:59:33.484 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_21 2026-02-22T21:59:33.484 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T21:59:33.484 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:59:33.484 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:33.484 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_21 2026-02-22T21:59:33.484 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = journal ']' 2026-02-22T21:59:33.484 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T21:59:33.484 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ mirror_image_snapshot cluster2 mirror image_21 2026-02-22T21:59:33.485 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:59:33.485 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:33.485 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_21 2026-02-22T21:59:33.485 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_21 2026-02-22T21:59:34.239 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Snapshot ID: 126 2026-02-22T21:59:34.244 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_21 2026-02-22T21:59:34.245 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T21:59:34.245 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:59:34.245 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:34.245 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_21 2026-02-22T21:59:34.245 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster2-mirror-image_21.status 2026-02-22T21:59:34.245 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster2-mirror-image_21.status 2026-02-22T21:59:34.245 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T21:59:34.246 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_log=/tmp/tmp.j44LZPowal/cluster2-mirror-image_21.status 2026-02-22T21:59:34.246 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster1-mirror-image_21.status 2026-02-22T21:59:34.246 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster1-mirror-image_21.status 2026-02-22T21:59:34.246 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T21:59:34.248 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_status_log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_21.status 2026-02-22T21:59:34.248 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_21 /tmp/tmp.j44LZPowal/cluster2-mirror-image_21.status 2026-02-22T21:59:34.248 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:59:34.248 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:34.248 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_21 2026-02-22T21:59:34.249 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster2-mirror-image_21.status 2026-02-22T21:59:34.249 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_21 --format xml 2026-02-22T21:59:34.249 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:59:34.276 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T21:59:34.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snapshot_id=126 2026-02-22T21:59:34.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:59:34.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:59:34.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.2 2026-02-22T21:59:34.479 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_21 /tmp/tmp.j44LZPowal/cluster1-mirror-image_21.status 2026-02-22T21:59:34.480 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:59:34.480 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:34.480 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_21 2026-02-22T21:59:34.480 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_21.status 2026-02-22T21:59:34.480 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_21 --format xml 2026-02-22T21:59:34.480 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:59:34.509 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:59:34.510 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id=55 2026-02-22T21:59:34.510 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 126 = 55 2026-02-22T21:59:34.510 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:59:34.511 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.4 2026-02-22T21:59:34.912 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_21 /tmp/tmp.j44LZPowal/cluster1-mirror-image_21.status 2026-02-22T21:59:34.912 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:59:34.912 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:34.912 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_21 2026-02-22T21:59:34.912 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_21.status 2026-02-22T21:59:34.912 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_21 --format xml 2026-02-22T21:59:34.912 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:59:34.941 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:59:34.942 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:59:34.944 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T21:59:34.944 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T21:59:34.944 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T21:59:34.944 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T21:59:34.944 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 126 = '' 2026-02-22T21:59:34.944 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:59:34.944 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.8 2026-02-22T21:59:35.744 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_21 /tmp/tmp.j44LZPowal/cluster1-mirror-image_21.status 2026-02-22T21:59:35.744 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:59:35.744 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:35.744 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_21 2026-02-22T21:59:35.744 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_21.status 2026-02-22T21:59:35.744 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_21 --format xml 2026-02-22T21:59:35.745 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:59:35.778 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:59:35.780 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id=126 2026-02-22T21:59:35.780 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 126 = 126 2026-02-22T21:59:35.780 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:59:35.780 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snap_present cluster1 mirror image_21 snap 2026-02-22T21:59:35.780 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:59:35.780 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:35.780 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_21 2026-02-22T21:59:35.780 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T21:59:35.780 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T21:59:35.780 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T21:59:35.780 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T21:59:36.781 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 info mirror/image_21@snap 2026-02-22T21:59:36.807 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_21': 2026-02-22T21:59:36.807 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 128 MiB in 32 objects 2026-02-22T21:59:36.807 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T21:59:36.807 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 2 2026-02-22T21:59:36.808 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 119ac248c4d7 2026-02-22T21:59:36.808 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.119ac248c4d7 2026-02-22T21:59:36.808 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T21:59:36.808 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock, non-primary 2026-02-22T21:59:36.808 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T21:59:36.808 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T21:59:36.808 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:51:25 2026 2026-02-22T21:59:36.808 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:51:25 2026 2026-02-22T21:59:36.808 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:51:25 2026 2026-02-22T21:59:36.808 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: protected: False 2026-02-22T21:59:36.808 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T21:59:36.808 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T21:59:36.808 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: fcfc28ec-1e7e-4846-a752-e36745d49c74 2026-02-22T21:59:36.808 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: false 2026-02-22T21:59:36.809 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:59:36.809 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ compare_image_snaps mirror image_21 snap 2026-02-22T21:59:36.809 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:36.809 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_21 2026-02-22T21:59:36.809 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T21:59:36.809 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local ret=0 2026-02-22T21:59:36.809 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local rmt_export=/tmp/tmp.j44LZPowal/cluster2-mirror-image_21.export 2026-02-22T21:59:36.809 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local loc_export=/tmp/tmp.j44LZPowal/cluster1-mirror-image_21.export 2026-02-22T21:59:36.809 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-image_21.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_21.export 2026-02-22T21:59:36.810 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror export image_21@snap /tmp/tmp.j44LZPowal/cluster2-mirror-image_21.export 2026-02-22T21:59:36.921 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T21:59:36.929 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 -p mirror export image_21@snap /tmp/tmp.j44LZPowal/cluster1-mirror-image_21.export 2026-02-22T21:59:37.051 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T21:59:37.059 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ cmp /tmp/tmp.j44LZPowal/cluster2-mirror-image_21.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_21.export 2026-02-22T21:59:37.109 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-image_21.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_21.export 2026-02-22T21:59:37.142 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:59:37.142 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T21:59:37.142 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_22 2026-02-22T21:59:37.142 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_snap cluster2 mirror image_22 snap 2026-02-22T21:59:37.142 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:59:37.142 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:37.143 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_22 2026-02-22T21:59:37.143 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T21:59:37.143 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror snap create image_22@snap --debug-rbd=20 --debug-journaler=20 2026-02-22T21:59:39.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_started cluster1 mirror image_22 2026-02-22T21:59:39.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:59:39.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:39.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_22 2026-02-22T21:59:39.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_state cluster1 mirror image_22 started 2026-02-22T21:59:39.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:59:39.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:39.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_22 2026-02-22T21:59:39.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=started 2026-02-22T21:59:39.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T21:59:39.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:59:39.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T21:59:40.183 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_replay_state cluster1 mirror image_22 started 2026-02-22T21:59:40.183 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:59:40.183 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:40.183 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_22 2026-02-22T21:59:40.183 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=started 2026-02-22T21:59:40.183 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_result 2026-02-22T21:59:40.184 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=stopped 2026-02-22T21:59:40.184 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_22 2026-02-22T21:59:40.184 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_instance=cluster1 2026-02-22T21:59:40.184 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ grep -i state 2026-02-22T21:59:40.184 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:59:40.184 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T21:59:40.184 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance=cluster1 2026-02-22T21:59:40.184 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local loop_instance 2026-02-22T21:59:40.184 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:59:40.184 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sleep 0 2026-02-22T21:59:40.187 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:59:40.188 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ seq 0 3 2026-02-22T21:59:40.188 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:59:40.188 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_22 2026-02-22T21:59:40.188 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:0 2026-02-22T21:59:40.188 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:59:40.188 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:59:40.188 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:59:40.188 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:0 2026-02-22T21:59:40.188 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:59:40.188 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:59:40.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:59:40.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=0 2026-02-22T21:59:40.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:59:40.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:59:40.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:59:40.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=0 2026-02-22T21:59:40.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T21:59:40.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:59:40.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:59:40.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:59:40.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:59:40.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:59:40.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:0 2026-02-22T21:59:40.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:59:40.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:59:40.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:59:40.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T21:59:40.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:59:40.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:59:40.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:59:40.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=0 2026-02-22T21:59:40.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=0 2026-02-22T21:59:40.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:59:40.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:59:40.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:59:40.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:59:40.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_22 2026-02-22T21:59:40.243 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_22'] 2026-02-22T21:59:40.243 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:59:40.243 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T21:59:40.246 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:59:40.246 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_22 2026-02-22T21:59:40.247 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:1 2026-02-22T21:59:40.247 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:59:40.247 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:59:40.247 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T21:59:40.247 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:1 2026-02-22T21:59:40.247 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:59:40.247 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:59:40.247 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:59:40.247 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=1 2026-02-22T21:59:40.247 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T21:59:40.247 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:59:40.248 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:59:40.248 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=1 2026-02-22T21:59:40.248 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T21:59:40.248 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T21:59:40.248 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:1 2026-02-22T21:59:40.248 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:59:40.248 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:59:40.248 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T21:59:40.248 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:1 2026-02-22T21:59:40.248 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:59:40.248 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:59:40.248 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:59:40.248 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T21:59:40.249 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T21:59:40.249 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:59:40.249 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:59:40.249 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=1 2026-02-22T21:59:40.249 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=1 2026-02-22T21:59:40.249 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T21:59:40.256 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:59:40.256 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:59:40.257 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:59:40.257 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_22 2026-02-22T21:59:40.308 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ return 0 2026-02-22T21:59:40.309 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:59:40.309 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:59:40.309 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep -i Replaying 2026-02-22T21:59:40.310 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: "state": "Replaying" 2026-02-22T21:59:40.311 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_state=started 2026-02-22T21:59:40.311 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test started = started 2026-02-22T21:59:40.311 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:59:40.311 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_22 2026-02-22T21:59:40.312 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T21:59:40.312 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:59:40.312 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:40.312 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_22 2026-02-22T21:59:40.312 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = journal ']' 2026-02-22T21:59:40.312 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T21:59:40.312 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ mirror_image_snapshot cluster2 mirror image_22 2026-02-22T21:59:40.312 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:59:40.312 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:40.312 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_22 2026-02-22T21:59:40.312 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_22 2026-02-22T21:59:41.183 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Snapshot ID: 128 2026-02-22T21:59:41.188 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_22 2026-02-22T21:59:41.188 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T21:59:41.188 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:59:41.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:41.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_22 2026-02-22T21:59:41.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster2-mirror-image_22.status 2026-02-22T21:59:41.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster2-mirror-image_22.status 2026-02-22T21:59:41.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T21:59:41.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_log=/tmp/tmp.j44LZPowal/cluster2-mirror-image_22.status 2026-02-22T21:59:41.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster1-mirror-image_22.status 2026-02-22T21:59:41.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster1-mirror-image_22.status 2026-02-22T21:59:41.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T21:59:41.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_status_log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_22.status 2026-02-22T21:59:41.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_22 /tmp/tmp.j44LZPowal/cluster2-mirror-image_22.status 2026-02-22T21:59:41.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:59:41.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:41.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_22 2026-02-22T21:59:41.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster2-mirror-image_22.status 2026-02-22T21:59:41.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_22 --format xml 2026-02-22T21:59:41.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:59:41.219 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T21:59:41.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snapshot_id=128 2026-02-22T21:59:41.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:59:41.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:59:41.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.2 2026-02-22T21:59:41.422 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_22 /tmp/tmp.j44LZPowal/cluster1-mirror-image_22.status 2026-02-22T21:59:41.422 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:59:41.422 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:41.423 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_22 2026-02-22T21:59:41.423 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_22.status 2026-02-22T21:59:41.423 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_22 --format xml 2026-02-22T21:59:41.423 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:59:41.454 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:59:41.456 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id=56 2026-02-22T21:59:41.456 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 128 = 56 2026-02-22T21:59:41.456 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:59:41.456 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.4 2026-02-22T21:59:41.857 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_22 /tmp/tmp.j44LZPowal/cluster1-mirror-image_22.status 2026-02-22T21:59:41.857 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:59:41.857 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:41.857 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_22 2026-02-22T21:59:41.857 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_22.status 2026-02-22T21:59:41.857 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_22 --format xml 2026-02-22T21:59:41.857 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:59:41.888 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:59:41.888 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:59:41.889 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T21:59:41.889 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T21:59:41.889 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T21:59:41.890 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T21:59:41.890 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 128 = '' 2026-02-22T21:59:41.890 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:59:41.890 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.8 2026-02-22T21:59:42.690 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_22 /tmp/tmp.j44LZPowal/cluster1-mirror-image_22.status 2026-02-22T21:59:42.691 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:59:42.691 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:42.691 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_22 2026-02-22T21:59:42.691 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_22.status 2026-02-22T21:59:42.691 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_22 --format xml 2026-02-22T21:59:42.691 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:59:42.723 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:59:42.725 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id=128 2026-02-22T21:59:42.725 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 128 = 128 2026-02-22T21:59:42.725 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:59:42.725 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snap_present cluster1 mirror image_22 snap 2026-02-22T21:59:42.725 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:59:42.725 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:42.725 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_22 2026-02-22T21:59:42.725 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T21:59:42.725 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T21:59:42.726 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T21:59:42.726 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T21:59:43.725 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 info mirror/image_22@snap 2026-02-22T21:59:43.752 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_22': 2026-02-22T21:59:43.753 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 128 MiB in 32 objects 2026-02-22T21:59:43.753 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T21:59:43.753 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 2 2026-02-22T21:59:43.753 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 11bd1ef1a1d 2026-02-22T21:59:43.753 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.11bd1ef1a1d 2026-02-22T21:59:43.753 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T21:59:43.753 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock, non-primary 2026-02-22T21:59:43.753 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T21:59:43.753 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T21:59:43.754 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:51:31 2026 2026-02-22T21:59:43.754 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:51:31 2026 2026-02-22T21:59:43.754 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:51:31 2026 2026-02-22T21:59:43.754 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: protected: False 2026-02-22T21:59:43.754 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T21:59:43.754 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T21:59:43.754 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: cbed6911-da61-47b0-a3cc-2ddeb6616c4c 2026-02-22T21:59:43.754 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: false 2026-02-22T21:59:43.756 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:59:43.756 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ compare_image_snaps mirror image_22 snap 2026-02-22T21:59:43.756 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:43.756 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_22 2026-02-22T21:59:43.756 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T21:59:43.756 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local ret=0 2026-02-22T21:59:43.756 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local rmt_export=/tmp/tmp.j44LZPowal/cluster2-mirror-image_22.export 2026-02-22T21:59:43.756 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local loc_export=/tmp/tmp.j44LZPowal/cluster1-mirror-image_22.export 2026-02-22T21:59:43.756 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-image_22.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_22.export 2026-02-22T21:59:43.757 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror export image_22@snap /tmp/tmp.j44LZPowal/cluster2-mirror-image_22.export 2026-02-22T21:59:43.858 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T21:59:43.866 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 -p mirror export image_22@snap /tmp/tmp.j44LZPowal/cluster1-mirror-image_22.export 2026-02-22T21:59:44.006 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T21:59:44.013 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ cmp /tmp/tmp.j44LZPowal/cluster2-mirror-image_22.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_22.export 2026-02-22T21:59:44.063 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-image_22.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_22.export 2026-02-22T21:59:44.098 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:59:44.098 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T21:59:44.098 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_23 2026-02-22T21:59:44.098 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_snap cluster2 mirror image_23 snap 2026-02-22T21:59:44.098 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:59:44.098 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:44.098 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_23 2026-02-22T21:59:44.098 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T21:59:44.098 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror snap create image_23@snap --debug-rbd=20 --debug-journaler=20 2026-02-22T21:59:45.188 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_started cluster1 mirror image_23 2026-02-22T21:59:45.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:59:45.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:45.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_23 2026-02-22T21:59:45.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_state cluster1 mirror image_23 started 2026-02-22T21:59:45.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:59:45.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:45.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_23 2026-02-22T21:59:45.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=started 2026-02-22T21:59:45.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T21:59:45.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:59:45.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T21:59:46.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_replay_state cluster1 mirror image_23 started 2026-02-22T21:59:46.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:59:46.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:46.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_23 2026-02-22T21:59:46.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=started 2026-02-22T21:59:46.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_result 2026-02-22T21:59:46.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=stopped 2026-02-22T21:59:46.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_23 2026-02-22T21:59:46.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ grep -i state 2026-02-22T21:59:46.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_instance=cluster1 2026-02-22T21:59:46.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:59:46.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T21:59:46.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance=cluster1 2026-02-22T21:59:46.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local loop_instance 2026-02-22T21:59:46.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:59:46.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sleep 0 2026-02-22T21:59:46.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:59:46.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ seq 0 3 2026-02-22T21:59:46.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:59:46.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_23 2026-02-22T21:59:46.195 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:0 2026-02-22T21:59:46.195 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:59:46.195 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:59:46.195 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:59:46.195 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:0 2026-02-22T21:59:46.195 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:59:46.195 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:59:46.195 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:59:46.195 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=0 2026-02-22T21:59:46.195 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:59:46.195 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:59:46.195 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:59:46.196 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=0 2026-02-22T21:59:46.196 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T21:59:46.196 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:59:46.196 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:59:46.196 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:59:46.196 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:59:46.196 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:59:46.196 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:0 2026-02-22T21:59:46.196 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:59:46.196 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:59:46.196 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:59:46.196 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T21:59:46.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:59:46.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:59:46.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:59:46.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=0 2026-02-22T21:59:46.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=0 2026-02-22T21:59:46.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:59:46.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:59:46.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:59:46.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:59:46.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_23 2026-02-22T21:59:46.258 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ return 0 2026-02-22T21:59:46.258 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:59:46.259 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:59:46.259 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep -i Replaying 2026-02-22T21:59:46.260 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: "state": "Replaying" 2026-02-22T21:59:46.261 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_state=started 2026-02-22T21:59:46.261 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test started = started 2026-02-22T21:59:46.261 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:59:46.261 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_23 2026-02-22T21:59:46.262 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T21:59:46.262 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:59:46.262 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:46.262 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_23 2026-02-22T21:59:46.262 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = journal ']' 2026-02-22T21:59:46.262 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T21:59:46.262 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ mirror_image_snapshot cluster2 mirror image_23 2026-02-22T21:59:46.262 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:59:46.262 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:46.262 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_23 2026-02-22T21:59:46.262 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_23 2026-02-22T21:59:47.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Snapshot ID: 130 2026-02-22T21:59:47.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_23 2026-02-22T21:59:47.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T21:59:47.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:59:47.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:47.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_23 2026-02-22T21:59:47.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster2-mirror-image_23.status 2026-02-22T21:59:47.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster2-mirror-image_23.status 2026-02-22T21:59:47.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T21:59:47.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_log=/tmp/tmp.j44LZPowal/cluster2-mirror-image_23.status 2026-02-22T21:59:47.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster1-mirror-image_23.status 2026-02-22T21:59:47.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster1-mirror-image_23.status 2026-02-22T21:59:47.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T21:59:47.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_status_log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_23.status 2026-02-22T21:59:47.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_23 /tmp/tmp.j44LZPowal/cluster2-mirror-image_23.status 2026-02-22T21:59:47.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:59:47.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:47.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_23 2026-02-22T21:59:47.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster2-mirror-image_23.status 2026-02-22T21:59:47.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_23 --format xml 2026-02-22T21:59:47.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:59:47.229 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T21:59:47.230 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snapshot_id=130 2026-02-22T21:59:47.230 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:59:47.231 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:59:47.231 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.2 2026-02-22T21:59:47.432 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_23 /tmp/tmp.j44LZPowal/cluster1-mirror-image_23.status 2026-02-22T21:59:47.432 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:59:47.432 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:47.432 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_23 2026-02-22T21:59:47.432 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_23.status 2026-02-22T21:59:47.433 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_23 --format xml 2026-02-22T21:59:47.433 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:59:47.462 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:59:47.463 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:59:47.463 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T21:59:47.464 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T21:59:47.464 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T21:59:47.464 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T21:59:47.464 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 130 = '' 2026-02-22T21:59:47.465 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:59:47.465 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.4 2026-02-22T21:59:47.865 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_23 /tmp/tmp.j44LZPowal/cluster1-mirror-image_23.status 2026-02-22T21:59:47.866 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:59:47.866 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:47.866 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_23 2026-02-22T21:59:47.866 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_23.status 2026-02-22T21:59:47.866 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_23 --format xml 2026-02-22T21:59:47.866 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:59:47.895 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:59:47.895 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:59:47.896 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T21:59:47.896 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T21:59:47.896 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T21:59:47.897 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T21:59:47.897 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 130 = '' 2026-02-22T21:59:47.898 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:59:47.898 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.8 2026-02-22T21:59:48.698 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_23 /tmp/tmp.j44LZPowal/cluster1-mirror-image_23.status 2026-02-22T21:59:48.698 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:59:48.699 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:48.699 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_23 2026-02-22T21:59:48.699 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_23.status 2026-02-22T21:59:48.699 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_23 --format xml 2026-02-22T21:59:48.699 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:59:48.731 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:59:48.734 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id=130 2026-02-22T21:59:48.734 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 130 = 130 2026-02-22T21:59:48.734 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:59:48.734 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snap_present cluster1 mirror image_23 snap 2026-02-22T21:59:48.734 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:59:48.734 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:48.734 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_23 2026-02-22T21:59:48.734 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T21:59:48.734 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T21:59:48.734 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T21:59:48.735 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T21:59:49.734 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 info mirror/image_23@snap 2026-02-22T21:59:49.761 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_23': 2026-02-22T21:59:49.762 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 128 MiB in 32 objects 2026-02-22T21:59:49.762 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T21:59:49.762 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 2 2026-02-22T21:59:49.762 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 12275e3bece 2026-02-22T21:59:49.762 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.12275e3bece 2026-02-22T21:59:49.762 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T21:59:49.762 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock, non-primary 2026-02-22T21:59:49.762 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T21:59:49.762 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T21:59:49.763 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:51:46 2026 2026-02-22T21:59:49.763 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:51:46 2026 2026-02-22T21:59:49.763 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:51:46 2026 2026-02-22T21:59:49.763 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: protected: False 2026-02-22T21:59:49.763 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T21:59:49.763 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T21:59:49.763 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: 707a358f-f62b-47ff-a4e3-ae016127a25e 2026-02-22T21:59:49.763 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: false 2026-02-22T21:59:49.764 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:59:49.764 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ compare_image_snaps mirror image_23 snap 2026-02-22T21:59:49.764 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:49.764 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_23 2026-02-22T21:59:49.765 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T21:59:49.765 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local ret=0 2026-02-22T21:59:49.765 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local rmt_export=/tmp/tmp.j44LZPowal/cluster2-mirror-image_23.export 2026-02-22T21:59:49.765 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local loc_export=/tmp/tmp.j44LZPowal/cluster1-mirror-image_23.export 2026-02-22T21:59:49.765 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-image_23.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_23.export 2026-02-22T21:59:49.765 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror export image_23@snap /tmp/tmp.j44LZPowal/cluster2-mirror-image_23.export 2026-02-22T21:59:49.876 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T21:59:49.882 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 -p mirror export image_23@snap /tmp/tmp.j44LZPowal/cluster1-mirror-image_23.export 2026-02-22T21:59:50.018 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T21:59:50.026 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ cmp /tmp/tmp.j44LZPowal/cluster2-mirror-image_23.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_23.export 2026-02-22T21:59:50.073 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-image_23.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_23.export 2026-02-22T21:59:50.108 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:59:50.108 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T21:59:50.109 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_24 2026-02-22T21:59:50.109 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_snap cluster2 mirror image_24 snap 2026-02-22T21:59:50.109 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:59:50.109 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:50.109 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_24 2026-02-22T21:59:50.109 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T21:59:50.109 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror snap create image_24@snap --debug-rbd=20 --debug-journaler=20 2026-02-22T21:59:51.196 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_started cluster1 mirror image_24 2026-02-22T21:59:51.196 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:59:51.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:51.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_24 2026-02-22T21:59:51.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_state cluster1 mirror image_24 started 2026-02-22T21:59:51.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:59:51.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:51.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_24 2026-02-22T21:59:51.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=started 2026-02-22T21:59:51.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T21:59:51.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:59:51.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T21:59:52.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_replay_state cluster1 mirror image_24 started 2026-02-22T21:59:52.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:59:52.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:52.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_24 2026-02-22T21:59:52.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=started 2026-02-22T21:59:52.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_result 2026-02-22T21:59:52.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=stopped 2026-02-22T21:59:52.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_24 2026-02-22T21:59:52.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_instance=cluster1 2026-02-22T21:59:52.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:59:52.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T21:59:52.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance=cluster1 2026-02-22T21:59:52.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local loop_instance 2026-02-22T21:59:52.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:59:52.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sleep 0 2026-02-22T21:59:52.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ grep -i state 2026-02-22T21:59:52.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:59:52.203 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ seq 0 3 2026-02-22T21:59:52.203 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:59:52.203 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_24 2026-02-22T21:59:52.203 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:0 2026-02-22T21:59:52.203 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:59:52.203 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:59:52.203 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:59:52.203 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:0 2026-02-22T21:59:52.203 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:59:52.203 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:59:52.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:59:52.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=0 2026-02-22T21:59:52.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:59:52.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:59:52.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:59:52.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=0 2026-02-22T21:59:52.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T21:59:52.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:59:52.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:59:52.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:59:52.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:59:52.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:59:52.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:0 2026-02-22T21:59:52.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:59:52.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:59:52.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:59:52.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T21:59:52.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:59:52.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:59:52.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:59:52.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=0 2026-02-22T21:59:52.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=0 2026-02-22T21:59:52.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:59:52.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:59:52.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:59:52.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:59:52.213 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_24 2026-02-22T21:59:52.258 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_24'] 2026-02-22T21:59:52.258 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:59:52.258 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T21:59:52.262 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:59:52.262 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_24 2026-02-22T21:59:52.262 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:1 2026-02-22T21:59:52.262 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:59:52.262 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:59:52.262 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T21:59:52.262 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:1 2026-02-22T21:59:52.262 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:59:52.262 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:59:52.262 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:59:52.263 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=1 2026-02-22T21:59:52.263 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T21:59:52.263 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:59:52.263 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:59:52.263 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=1 2026-02-22T21:59:52.263 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T21:59:52.263 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T21:59:52.263 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:1 2026-02-22T21:59:52.263 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:59:52.263 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:59:52.263 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T21:59:52.263 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:1 2026-02-22T21:59:52.264 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:59:52.264 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:59:52.264 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:59:52.264 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T21:59:52.264 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T21:59:52.264 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:59:52.264 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:59:52.264 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=1 2026-02-22T21:59:52.264 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=1 2026-02-22T21:59:52.264 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T21:59:52.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:59:52.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:59:52.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:59:52.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_24 2026-02-22T21:59:52.318 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_24'] 2026-02-22T21:59:52.318 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:59:52.318 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T21:59:52.321 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:59:52.321 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_24 2026-02-22T21:59:52.321 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:2 2026-02-22T21:59:52.321 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:59:52.321 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:59:52.321 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T21:59:52.321 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:2 2026-02-22T21:59:52.321 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:59:52.321 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:59:52.321 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:59:52.322 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=2 2026-02-22T21:59:52.322 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T21:59:52.322 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:59:52.322 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:59:52.322 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=2 2026-02-22T21:59:52.322 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T21:59:52.322 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T21:59:52.322 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:2 2026-02-22T21:59:52.322 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:59:52.322 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:59:52.322 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T21:59:52.322 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:2 2026-02-22T21:59:52.323 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:59:52.323 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:59:52.323 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:59:52.323 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T21:59:52.323 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T21:59:52.323 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:59:52.323 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:59:52.323 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=2 2026-02-22T21:59:52.323 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=2 2026-02-22T21:59:52.324 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T21:59:52.331 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:59:52.331 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:59:52.331 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:59:52.331 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_24 2026-02-22T21:59:52.377 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_24'] 2026-02-22T21:59:52.378 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:59:52.378 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T21:59:52.380 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:59:52.381 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_24 2026-02-22T21:59:52.381 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:3 2026-02-22T21:59:52.381 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:59:52.381 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:59:52.381 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T21:59:52.381 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:3 2026-02-22T21:59:52.381 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:59:52.381 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:59:52.381 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:59:52.381 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=3 2026-02-22T21:59:52.381 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T21:59:52.382 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:59:52.382 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:59:52.382 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=3 2026-02-22T21:59:52.382 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=3 2026-02-22T21:59:52.382 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T21:59:52.382 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:3 2026-02-22T21:59:52.382 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:59:52.382 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:59:52.382 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T21:59:52.382 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:3 2026-02-22T21:59:52.382 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:59:52.382 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:59:52.382 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:59:52.383 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=3 2026-02-22T21:59:52.383 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T21:59:52.383 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:59:52.383 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:59:52.383 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=3 2026-02-22T21:59:52.383 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=3 2026-02-22T21:59:52.383 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T21:59:52.390 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:59:52.391 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:59:52.391 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:59:52.391 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_24 2026-02-22T21:59:52.442 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ return 0 2026-02-22T21:59:52.443 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:59:52.443 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:59:52.443 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep -i Replaying 2026-02-22T21:59:52.443 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: "state": "Replaying" 2026-02-22T21:59:52.445 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_state=started 2026-02-22T21:59:52.445 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test started = started 2026-02-22T21:59:52.445 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:59:52.445 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_24 2026-02-22T21:59:52.445 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T21:59:52.445 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:59:52.445 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:52.445 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_24 2026-02-22T21:59:52.445 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = journal ']' 2026-02-22T21:59:52.446 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T21:59:52.446 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ mirror_image_snapshot cluster2 mirror image_24 2026-02-22T21:59:52.446 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:59:52.446 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:52.446 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_24 2026-02-22T21:59:52.446 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_24 2026-02-22T21:59:53.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Snapshot ID: 132 2026-02-22T21:59:53.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_24 2026-02-22T21:59:53.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T21:59:53.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:59:53.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:53.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_24 2026-02-22T21:59:53.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster2-mirror-image_24.status 2026-02-22T21:59:53.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster2-mirror-image_24.status 2026-02-22T21:59:53.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T21:59:53.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_log=/tmp/tmp.j44LZPowal/cluster2-mirror-image_24.status 2026-02-22T21:59:53.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster1-mirror-image_24.status 2026-02-22T21:59:53.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster1-mirror-image_24.status 2026-02-22T21:59:53.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T21:59:53.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_status_log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_24.status 2026-02-22T21:59:53.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_24 /tmp/tmp.j44LZPowal/cluster2-mirror-image_24.status 2026-02-22T21:59:53.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:59:53.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:53.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_24 2026-02-22T21:59:53.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster2-mirror-image_24.status 2026-02-22T21:59:53.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_24 --format xml 2026-02-22T21:59:53.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:59:53.234 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T21:59:53.236 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snapshot_id=132 2026-02-22T21:59:53.236 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:59:53.236 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:59:53.237 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.2 2026-02-22T21:59:53.437 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_24 /tmp/tmp.j44LZPowal/cluster1-mirror-image_24.status 2026-02-22T21:59:53.437 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:59:53.437 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:53.437 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_24 2026-02-22T21:59:53.437 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_24.status 2026-02-22T21:59:53.437 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_24 --format xml 2026-02-22T21:59:53.437 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:59:53.467 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:59:53.467 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:59:53.468 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T21:59:53.468 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T21:59:53.468 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T21:59:53.469 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T21:59:53.469 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 132 = '' 2026-02-22T21:59:53.469 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:59:53.469 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.4 2026-02-22T21:59:53.870 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_24 /tmp/tmp.j44LZPowal/cluster1-mirror-image_24.status 2026-02-22T21:59:53.870 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:59:53.870 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:53.870 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_24 2026-02-22T21:59:53.870 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_24.status 2026-02-22T21:59:53.871 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_24 --format xml 2026-02-22T21:59:53.871 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:59:53.898 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:59:53.899 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:59:53.900 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T21:59:53.900 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T21:59:53.900 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T21:59:53.900 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T21:59:53.901 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 132 = '' 2026-02-22T21:59:53.901 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:59:53.901 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.8 2026-02-22T21:59:54.702 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_24 /tmp/tmp.j44LZPowal/cluster1-mirror-image_24.status 2026-02-22T21:59:54.702 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:59:54.702 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:54.702 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_24 2026-02-22T21:59:54.702 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_24.status 2026-02-22T21:59:54.702 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_24 --format xml 2026-02-22T21:59:54.702 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:59:54.733 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:59:54.734 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id=132 2026-02-22T21:59:54.735 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 132 = 132 2026-02-22T21:59:54.735 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:59:54.735 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snap_present cluster1 mirror image_24 snap 2026-02-22T21:59:54.735 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:59:54.735 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:54.735 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_24 2026-02-22T21:59:54.735 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T21:59:54.735 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T21:59:54.735 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T21:59:54.735 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T21:59:55.735 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 info mirror/image_24@snap 2026-02-22T21:59:55.763 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_24': 2026-02-22T21:59:55.763 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 128 MiB in 32 objects 2026-02-22T21:59:55.763 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T21:59:55.763 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 2 2026-02-22T21:59:55.763 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 12481ea229f8 2026-02-22T21:59:55.763 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.12481ea229f8 2026-02-22T21:59:55.763 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T21:59:55.763 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock, non-primary 2026-02-22T21:59:55.763 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T21:59:55.764 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T21:59:55.764 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:51:58 2026 2026-02-22T21:59:55.764 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:51:58 2026 2026-02-22T21:59:55.764 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:51:58 2026 2026-02-22T21:59:55.764 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: protected: False 2026-02-22T21:59:55.764 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T21:59:55.764 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T21:59:55.764 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: f06205bf-4d40-464a-9efe-4169410d86ad 2026-02-22T21:59:55.764 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: false 2026-02-22T21:59:55.766 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:59:55.766 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ compare_image_snaps mirror image_24 snap 2026-02-22T21:59:55.766 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:55.766 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_24 2026-02-22T21:59:55.766 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T21:59:55.766 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local ret=0 2026-02-22T21:59:55.766 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local rmt_export=/tmp/tmp.j44LZPowal/cluster2-mirror-image_24.export 2026-02-22T21:59:55.766 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local loc_export=/tmp/tmp.j44LZPowal/cluster1-mirror-image_24.export 2026-02-22T21:59:55.766 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-image_24.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_24.export 2026-02-22T21:59:55.766 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror export image_24@snap /tmp/tmp.j44LZPowal/cluster2-mirror-image_24.export 2026-02-22T21:59:55.868 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T21:59:55.876 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 -p mirror export image_24@snap /tmp/tmp.j44LZPowal/cluster1-mirror-image_24.export 2026-02-22T21:59:56.004 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T21:59:56.011 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ cmp /tmp/tmp.j44LZPowal/cluster2-mirror-image_24.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_24.export 2026-02-22T21:59:56.061 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-image_24.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_24.export 2026-02-22T21:59:56.094 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:59:56.094 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T21:59:56.094 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_25 2026-02-22T21:59:56.094 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_snap cluster2 mirror image_25 snap 2026-02-22T21:59:56.094 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:59:56.094 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:56.095 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_25 2026-02-22T21:59:56.095 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T21:59:56.095 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror snap create image_25@snap --debug-rbd=20 --debug-journaler=20 2026-02-22T21:59:57.203 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_started cluster1 mirror image_25 2026-02-22T21:59:57.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:59:57.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:57.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_25 2026-02-22T21:59:57.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_state cluster1 mirror image_25 started 2026-02-22T21:59:57.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:59:57.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:57.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_25 2026-02-22T21:59:57.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=started 2026-02-22T21:59:57.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T21:59:57.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:59:57.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T21:59:58.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_replay_state cluster1 mirror image_25 started 2026-02-22T21:59:58.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:59:58.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:58.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_25 2026-02-22T21:59:58.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=started 2026-02-22T21:59:58.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_result 2026-02-22T21:59:58.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=stopped 2026-02-22T21:59:58.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_25 2026-02-22T21:59:58.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ grep -i state 2026-02-22T21:59:58.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_instance=cluster1 2026-02-22T21:59:58.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:59:58.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T21:59:58.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance=cluster1 2026-02-22T21:59:58.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local loop_instance 2026-02-22T21:59:58.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:59:58.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sleep 0 2026-02-22T21:59:58.209 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:59:58.209 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ seq 0 3 2026-02-22T21:59:58.209 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:59:58.209 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_25 2026-02-22T21:59:58.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:0 2026-02-22T21:59:58.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T21:59:58.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T21:59:58.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:59:58.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:0 2026-02-22T21:59:58.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T21:59:58.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T21:59:58.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T21:59:58.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=0 2026-02-22T21:59:58.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:59:58.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T21:59:58.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:59:58.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=0 2026-02-22T21:59:58.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T21:59:58.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:59:58.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:59:58.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T21:59:58.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T21:59:58.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:59:58.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:0 2026-02-22T21:59:58.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:59:58.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T21:59:58.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T21:59:58.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T21:59:58.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:59:58.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:59:58.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T21:59:58.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=0 2026-02-22T21:59:58.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=0 2026-02-22T21:59:58.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:59:58.220 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:59:58.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:59:58.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:59:58.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_25 2026-02-22T21:59:58.270 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ return 0 2026-02-22T21:59:58.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:59:58.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:59:58.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep -i Replaying 2026-02-22T21:59:58.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: "state": "Replaying" 2026-02-22T21:59:58.273 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_state=started 2026-02-22T21:59:58.273 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test started = started 2026-02-22T21:59:58.273 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T21:59:58.273 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_25 2026-02-22T21:59:58.273 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T21:59:58.273 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:59:58.273 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:58.274 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_25 2026-02-22T21:59:58.274 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = journal ']' 2026-02-22T21:59:58.274 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T21:59:58.274 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ mirror_image_snapshot cluster2 mirror image_25 2026-02-22T21:59:58.274 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:59:58.274 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:58.274 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_25 2026-02-22T21:59:58.274 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_25 2026-02-22T21:59:59.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Snapshot ID: 134 2026-02-22T21:59:59.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_25 2026-02-22T21:59:59.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T21:59:59.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:59:59.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:59.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_25 2026-02-22T21:59:59.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster2-mirror-image_25.status 2026-02-22T21:59:59.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster2-mirror-image_25.status 2026-02-22T21:59:59.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T21:59:59.213 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_log=/tmp/tmp.j44LZPowal/cluster2-mirror-image_25.status 2026-02-22T21:59:59.214 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster1-mirror-image_25.status 2026-02-22T21:59:59.214 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster1-mirror-image_25.status 2026-02-22T21:59:59.214 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T21:59:59.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_status_log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_25.status 2026-02-22T21:59:59.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_25 /tmp/tmp.j44LZPowal/cluster2-mirror-image_25.status 2026-02-22T21:59:59.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T21:59:59.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:59.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_25 2026-02-22T21:59:59.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster2-mirror-image_25.status 2026-02-22T21:59:59.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_25 --format xml 2026-02-22T21:59:59.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:59:59.243 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T21:59:59.245 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snapshot_id=134 2026-02-22T21:59:59.245 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:59:59.245 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:59:59.245 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.2 2026-02-22T21:59:59.446 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_25 /tmp/tmp.j44LZPowal/cluster1-mirror-image_25.status 2026-02-22T21:59:59.446 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:59:59.446 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:59.446 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_25 2026-02-22T21:59:59.446 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_25.status 2026-02-22T21:59:59.447 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_25 --format xml 2026-02-22T21:59:59.447 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:59:59.476 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:59:59.477 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:59:59.477 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T21:59:59.477 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T21:59:59.478 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T21:59:59.478 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T21:59:59.479 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 134 = '' 2026-02-22T21:59:59.479 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:59:59.479 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.4 2026-02-22T21:59:59.879 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_25 /tmp/tmp.j44LZPowal/cluster1-mirror-image_25.status 2026-02-22T21:59:59.879 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T21:59:59.879 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T21:59:59.879 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_25 2026-02-22T21:59:59.880 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_25.status 2026-02-22T21:59:59.880 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_25 --format xml 2026-02-22T21:59:59.880 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T21:59:59.909 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T21:59:59.910 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T21:59:59.910 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T21:59:59.910 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T21:59:59.911 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T21:59:59.911 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T21:59:59.911 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 134 = '' 2026-02-22T21:59:59.911 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:59:59.912 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.8 2026-02-22T22:00:00.712 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_25 /tmp/tmp.j44LZPowal/cluster1-mirror-image_25.status 2026-02-22T22:00:00.712 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:00:00.713 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:00:00.713 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_25 2026-02-22T22:00:00.713 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_25.status 2026-02-22T22:00:00.713 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_25 --format xml 2026-02-22T22:00:00.713 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:00:00.945 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T22:00:00.947 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id=134 2026-02-22T22:00:00.947 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 134 = 134 2026-02-22T22:00:00.947 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:00:00.947 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snap_present cluster1 mirror image_25 snap 2026-02-22T22:00:00.947 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:00:00.947 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:00:00.947 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_25 2026-02-22T22:00:00.947 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T22:00:00.947 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:00:00.948 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T22:00:00.948 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T22:00:01.947 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 info mirror/image_25@snap 2026-02-22T22:00:01.974 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_25': 2026-02-22T22:00:01.974 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 128 MiB in 32 objects 2026-02-22T22:00:01.974 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T22:00:01.974 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 2 2026-02-22T22:00:01.974 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 119a93f2e1cd 2026-02-22T22:00:01.974 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.119a93f2e1cd 2026-02-22T22:00:01.974 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T22:00:01.974 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock, non-primary 2026-02-22T22:00:01.974 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T22:00:01.974 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T22:00:01.974 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:52:07 2026 2026-02-22T22:00:01.975 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:52:07 2026 2026-02-22T22:00:01.975 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:52:07 2026 2026-02-22T22:00:01.975 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: protected: False 2026-02-22T22:00:01.975 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T22:00:01.975 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T22:00:01.975 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: 886eb68e-ef31-471e-9872-4fcf569230c4 2026-02-22T22:00:01.975 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: false 2026-02-22T22:00:01.976 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:00:01.976 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ compare_image_snaps mirror image_25 snap 2026-02-22T22:00:01.976 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:00:01.976 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_25 2026-02-22T22:00:01.976 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T22:00:01.976 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local ret=0 2026-02-22T22:00:01.976 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local rmt_export=/tmp/tmp.j44LZPowal/cluster2-mirror-image_25.export 2026-02-22T22:00:01.976 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local loc_export=/tmp/tmp.j44LZPowal/cluster1-mirror-image_25.export 2026-02-22T22:00:01.976 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-image_25.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_25.export 2026-02-22T22:00:01.976 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror export image_25@snap /tmp/tmp.j44LZPowal/cluster2-mirror-image_25.export 2026-02-22T22:00:02.085 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:00:02.091 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 -p mirror export image_25@snap /tmp/tmp.j44LZPowal/cluster1-mirror-image_25.export 2026-02-22T22:00:02.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:00:02.218 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ cmp /tmp/tmp.j44LZPowal/cluster2-mirror-image_25.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_25.export 2026-02-22T22:00:02.266 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-image_25.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_25.export 2026-02-22T22:00:02.282 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.8052113334268374) > weight (0.25) 2026-02-22T22:00:02.282 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.1 2026-02-22T22:00:02.282 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Sent signal 15 2026-02-22T22:00:02.282 INFO:tasks.rbd_mirror_thrash:waiting for 5.574994462765201 secs before reviving daemons 2026-02-22T22:00:02.283 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial026.stderr:2026-02-22T22:00:02.281+0000 7fe028096640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 (PID: 56126) UID: 1000 2026-02-22T22:00:02.302 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:00:02.302 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:00:02.302 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_26 2026-02-22T22:00:02.302 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_snap cluster2 mirror image_26 snap 2026-02-22T22:00:02.302 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:00:02.302 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:00:02.302 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_26 2026-02-22T22:00:02.302 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T22:00:02.303 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror snap create image_26@snap --debug-rbd=20 --debug-journaler=20 2026-02-22T22:00:03.402 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_started cluster1 mirror image_26 2026-02-22T22:00:03.402 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:00:03.402 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:00:03.402 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_26 2026-02-22T22:00:03.402 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_state cluster1 mirror image_26 started 2026-02-22T22:00:03.402 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:00:03.403 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:00:03.403 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_26 2026-02-22T22:00:03.403 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=started 2026-02-22T22:00:03.403 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:00:03.403 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:00:03.403 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T22:00:04.403 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_replay_state cluster1 mirror image_26 started 2026-02-22T22:00:04.404 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:00:04.404 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:00:04.404 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_26 2026-02-22T22:00:04.404 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=started 2026-02-22T22:00:04.404 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_result 2026-02-22T22:00:04.404 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=stopped 2026-02-22T22:00:04.404 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_26 2026-02-22T22:00:04.405 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_instance=cluster1 2026-02-22T22:00:04.405 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:00:04.405 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:00:04.405 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance=cluster1 2026-02-22T22:00:04.405 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local loop_instance 2026-02-22T22:00:04.405 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:00:04.405 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ grep -i state 2026-02-22T22:00:04.405 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sleep 0 2026-02-22T22:00:04.409 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:00:04.409 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ seq 0 3 2026-02-22T22:00:04.409 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:00:04.409 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_26 2026-02-22T22:00:04.409 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:0 2026-02-22T22:00:04.409 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:00:04.410 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T22:00:04.410 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:00:04.410 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:0 2026-02-22T22:00:04.410 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T22:00:04.410 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T22:00:04.410 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:00:04.410 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=0 2026-02-22T22:00:04.410 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:00:04.410 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T22:00:04.410 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:00:04.411 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=0 2026-02-22T22:00:04.411 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T22:00:04.411 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:00:04.411 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:00:04.411 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T22:00:04.411 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T22:00:04.411 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:00:04.411 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:0 2026-02-22T22:00:04.411 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:00:04.411 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T22:00:04.411 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:00:04.412 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T22:00:04.412 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:00:04.412 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:00:04.412 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T22:00:04.412 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=0 2026-02-22T22:00:04.412 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=0 2026-02-22T22:00:04.412 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:00:04.420 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:00:04.420 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:00:04.420 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:00:04.420 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_26 2026-02-22T22:00:04.465 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_26'] 2026-02-22T22:00:04.465 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:00:04.465 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T22:00:04.468 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:00:04.468 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_26 2026-02-22T22:00:04.469 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:1 2026-02-22T22:00:04.469 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:00:04.469 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T22:00:04.469 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:00:04.469 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:1 2026-02-22T22:00:04.469 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T22:00:04.469 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T22:00:04.469 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:00:04.469 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=1 2026-02-22T22:00:04.470 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:00:04.470 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T22:00:04.470 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:00:04.470 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=1 2026-02-22T22:00:04.470 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T22:00:04.470 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:00:04.470 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:1 2026-02-22T22:00:04.470 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T22:00:04.470 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T22:00:04.470 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:00:04.470 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:1 2026-02-22T22:00:04.471 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:00:04.471 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T22:00:04.471 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:00:04.471 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T22:00:04.471 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T22:00:04.471 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:00:04.471 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T22:00:04.471 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=1 2026-02-22T22:00:04.471 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=1 2026-02-22T22:00:04.471 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:00:04.478 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:00:04.478 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:00:04.478 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:00:04.479 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_26 2026-02-22T22:00:04.512 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-22T22:00:04.518 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:00:04.518 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_26 2026-02-22T22:00:04.518 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:2 2026-02-22T22:00:04.518 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:00:04.518 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T22:00:04.518 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:00:04.519 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:2 2026-02-22T22:00:04.519 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T22:00:04.519 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T22:00:04.519 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:00:04.519 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=2 2026-02-22T22:00:04.519 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:00:04.519 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T22:00:04.519 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:00:04.520 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=2 2026-02-22T22:00:04.520 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T22:00:04.520 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:00:04.520 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:2 2026-02-22T22:00:04.520 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T22:00:04.520 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T22:00:04.520 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:00:04.520 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:2 2026-02-22T22:00:04.521 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:00:04.521 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T22:00:04.521 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:00:04.521 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T22:00:04.521 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T22:00:04.521 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:00:04.521 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T22:00:04.521 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=2 2026-02-22T22:00:04.521 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=2 2026-02-22T22:00:04.522 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:00:04.525 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:00:04.525 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:00:04.526 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:00:04.526 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_26 2026-02-22T22:00:04.577 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ return 0 2026-02-22T22:00:04.577 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:00:04.577 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:00:04.577 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep -i Replaying 2026-02-22T22:00:04.578 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: "state": "Replaying" 2026-02-22T22:00:04.579 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_state=started 2026-02-22T22:00:04.579 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test started = started 2026-02-22T22:00:04.579 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:00:04.579 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_26 2026-02-22T22:00:04.579 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T22:00:04.579 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:00:04.579 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:00:04.579 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_26 2026-02-22T22:00:04.579 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = journal ']' 2026-02-22T22:00:04.579 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T22:00:04.579 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ mirror_image_snapshot cluster2 mirror image_26 2026-02-22T22:00:04.579 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:00:04.579 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:00:04.579 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_26 2026-02-22T22:00:04.579 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_26 2026-02-22T22:00:05.405 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Snapshot ID: 136 2026-02-22T22:00:05.411 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_26 2026-02-22T22:00:05.411 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T22:00:05.411 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:00:05.412 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:00:05.412 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_26 2026-02-22T22:00:05.412 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster2-mirror-image_26.status 2026-02-22T22:00:05.412 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster2-mirror-image_26.status 2026-02-22T22:00:05.412 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:00:05.413 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_log=/tmp/tmp.j44LZPowal/cluster2-mirror-image_26.status 2026-02-22T22:00:05.413 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster1-mirror-image_26.status 2026-02-22T22:00:05.413 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster1-mirror-image_26.status 2026-02-22T22:00:05.414 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:00:05.415 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_status_log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_26.status 2026-02-22T22:00:05.415 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_26 /tmp/tmp.j44LZPowal/cluster2-mirror-image_26.status 2026-02-22T22:00:05.415 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:00:05.415 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:00:05.415 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_26 2026-02-22T22:00:05.416 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster2-mirror-image_26.status 2026-02-22T22:00:05.416 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_26 --format xml 2026-02-22T22:00:05.416 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:00:05.442 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T22:00:05.444 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snapshot_id=136 2026-02-22T22:00:05.444 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T22:00:05.444 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:00:05.444 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.2 2026-02-22T22:00:05.645 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_26 /tmp/tmp.j44LZPowal/cluster1-mirror-image_26.status 2026-02-22T22:00:05.646 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:00:05.646 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:00:05.646 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_26 2026-02-22T22:00:05.646 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_26.status 2026-02-22T22:00:05.646 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_26 --format xml 2026-02-22T22:00:05.646 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:00:05.675 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T22:00:05.675 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T22:00:05.676 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T22:00:05.676 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T22:00:05.676 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T22:00:05.677 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T22:00:05.677 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 136 = '' 2026-02-22T22:00:05.677 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:00:05.677 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.4 2026-02-22T22:00:06.078 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_26 /tmp/tmp.j44LZPowal/cluster1-mirror-image_26.status 2026-02-22T22:00:06.078 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:00:06.078 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:00:06.078 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_26 2026-02-22T22:00:06.078 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_26.status 2026-02-22T22:00:06.078 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_26 --format xml 2026-02-22T22:00:06.078 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:00:06.109 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T22:00:06.110 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T22:00:06.111 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T22:00:06.111 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T22:00:06.111 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T22:00:06.111 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T22:00:06.111 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 136 = '' 2026-02-22T22:00:06.111 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:00:06.111 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.8 2026-02-22T22:00:06.912 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_26 /tmp/tmp.j44LZPowal/cluster1-mirror-image_26.status 2026-02-22T22:00:06.912 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:00:06.912 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:00:06.912 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_26 2026-02-22T22:00:06.912 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_26.status 2026-02-22T22:00:06.913 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_26 --format xml 2026-02-22T22:00:06.913 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:00:06.944 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T22:00:06.946 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id=136 2026-02-22T22:00:06.946 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 136 = 136 2026-02-22T22:00:06.946 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:00:06.946 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snap_present cluster1 mirror image_26 snap 2026-02-22T22:00:06.946 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:00:06.946 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:00:06.946 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_26 2026-02-22T22:00:06.946 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T22:00:06.946 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:00:06.946 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T22:00:06.946 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T22:00:07.858 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.1 2026-02-22T22:00:07.858 INFO:teuthology.orchestra.run:waiting for 600 2026-02-22T22:00:07.859 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.1 2026-02-22T22:00:07.859 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Restarting daemon 2026-02-22T22:00:07.859 DEBUG:teuthology.orchestra.run.trial026:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 2026-02-22T22:00:07.861 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Started 2026-02-22T22:00:07.862 INFO:tasks.rbd_mirror_thrash:waiting for 74.36400485762798 secs before thrashing 2026-02-22T22:00:07.947 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 info mirror/image_26@snap 2026-02-22T22:00:07.973 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_26': 2026-02-22T22:00:07.973 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 128 MiB in 32 objects 2026-02-22T22:00:07.973 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T22:00:07.973 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 2 2026-02-22T22:00:07.973 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 11bdcfb1215a 2026-02-22T22:00:07.973 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.11bdcfb1215a 2026-02-22T22:00:07.973 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T22:00:07.973 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock, non-primary 2026-02-22T22:00:07.973 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T22:00:07.973 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T22:00:07.973 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:52:19 2026 2026-02-22T22:00:07.973 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:52:19 2026 2026-02-22T22:00:07.973 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:52:19 2026 2026-02-22T22:00:07.973 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: protected: False 2026-02-22T22:00:07.973 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T22:00:07.973 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T22:00:07.973 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: f2b9fbe9-fa15-4b10-aead-5e1775df224b 2026-02-22T22:00:07.973 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: false 2026-02-22T22:00:07.974 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:00:07.974 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ compare_image_snaps mirror image_26 snap 2026-02-22T22:00:07.974 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:00:07.974 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_26 2026-02-22T22:00:07.975 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T22:00:07.975 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local ret=0 2026-02-22T22:00:07.975 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local rmt_export=/tmp/tmp.j44LZPowal/cluster2-mirror-image_26.export 2026-02-22T22:00:07.975 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local loc_export=/tmp/tmp.j44LZPowal/cluster1-mirror-image_26.export 2026-02-22T22:00:07.975 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-image_26.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_26.export 2026-02-22T22:00:07.975 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror export image_26@snap /tmp/tmp.j44LZPowal/cluster2-mirror-image_26.export 2026-02-22T22:00:08.078 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:00:08.086 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 -p mirror export image_26@snap /tmp/tmp.j44LZPowal/cluster1-mirror-image_26.export 2026-02-22T22:00:08.215 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:00:08.223 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ cmp /tmp/tmp.j44LZPowal/cluster2-mirror-image_26.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_26.export 2026-02-22T22:00:08.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-image_26.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_26.export 2026-02-22T22:00:08.305 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:00:08.306 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:00:08.306 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_27 2026-02-22T22:00:08.306 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_snap cluster2 mirror image_27 snap 2026-02-22T22:00:08.306 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:00:08.306 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:00:08.306 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_27 2026-02-22T22:00:08.306 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T22:00:08.306 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror snap create image_27@snap --debug-rbd=20 --debug-journaler=20 2026-02-22T22:00:09.409 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_started cluster1 mirror image_27 2026-02-22T22:00:09.409 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:00:09.410 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:00:09.410 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_27 2026-02-22T22:00:09.410 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_state cluster1 mirror image_27 started 2026-02-22T22:00:09.410 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:00:09.410 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:00:09.410 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_27 2026-02-22T22:00:09.410 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=started 2026-02-22T22:00:09.410 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:00:09.410 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:00:09.410 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T22:00:09.790 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.0 2026-02-22T22:00:09.791 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Sent signal 15 2026-02-22T22:00:09.791 INFO:tasks.rbd_mirror_thrash:waiting for 4.863954268065736 secs before reviving daemons 2026-02-22T22:00:09.791 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial026.stderr:2026-02-22T22:00:09.790+0000 7f3fe2a27640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 (PID: 65333) UID: 1000 2026-02-22T22:00:10.411 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_replay_state cluster1 mirror image_27 started 2026-02-22T22:00:10.411 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:00:10.411 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:00:10.411 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_27 2026-02-22T22:00:10.411 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=started 2026-02-22T22:00:10.411 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_result 2026-02-22T22:00:10.411 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=stopped 2026-02-22T22:00:10.412 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_27 2026-02-22T22:00:10.412 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ grep -i state 2026-02-22T22:00:10.412 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_instance=cluster1 2026-02-22T22:00:10.412 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:00:10.412 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:00:10.412 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance=cluster1 2026-02-22T22:00:10.412 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local loop_instance 2026-02-22T22:00:10.412 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:00:10.412 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sleep 0 2026-02-22T22:00:10.415 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:00:10.416 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ seq 0 3 2026-02-22T22:00:10.416 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:00:10.416 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_27 2026-02-22T22:00:10.416 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:0 2026-02-22T22:00:10.416 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:00:10.416 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T22:00:10.416 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:00:10.416 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:0 2026-02-22T22:00:10.416 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T22:00:10.417 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T22:00:10.417 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:00:10.417 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=0 2026-02-22T22:00:10.417 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:00:10.417 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T22:00:10.417 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:00:10.417 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=0 2026-02-22T22:00:10.417 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T22:00:10.417 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:00:10.417 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:00:10.417 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T22:00:10.417 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T22:00:10.418 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:00:10.418 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:0 2026-02-22T22:00:10.418 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:00:10.418 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T22:00:10.418 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:00:10.418 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T22:00:10.418 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:00:10.418 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:00:10.418 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T22:00:10.418 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=0 2026-02-22T22:00:10.418 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=0 2026-02-22T22:00:10.418 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:00:10.426 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:00:10.426 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:00:10.426 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:00:10.426 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_27 2026-02-22T22:00:10.475 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ return 0 2026-02-22T22:00:10.476 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:00:10.476 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:00:10.476 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep -i Replaying 2026-02-22T22:00:10.477 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: "state": "Replaying" 2026-02-22T22:00:10.478 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_state=started 2026-02-22T22:00:10.479 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test started = started 2026-02-22T22:00:10.479 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:00:10.479 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_27 2026-02-22T22:00:10.479 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T22:00:10.479 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:00:10.479 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:00:10.479 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_27 2026-02-22T22:00:10.479 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = journal ']' 2026-02-22T22:00:10.479 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T22:00:10.479 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ mirror_image_snapshot cluster2 mirror image_27 2026-02-22T22:00:10.479 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:00:10.480 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:00:10.480 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_27 2026-02-22T22:00:10.480 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_27 2026-02-22T22:00:11.414 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Snapshot ID: 138 2026-02-22T22:00:11.421 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_27 2026-02-22T22:00:11.421 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T22:00:11.421 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:00:11.421 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:00:11.422 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_27 2026-02-22T22:00:11.422 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster2-mirror-image_27.status 2026-02-22T22:00:11.422 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster2-mirror-image_27.status 2026-02-22T22:00:11.422 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:00:11.422 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_log=/tmp/tmp.j44LZPowal/cluster2-mirror-image_27.status 2026-02-22T22:00:11.423 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster1-mirror-image_27.status 2026-02-22T22:00:11.423 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster1-mirror-image_27.status 2026-02-22T22:00:11.423 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:00:11.425 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_status_log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_27.status 2026-02-22T22:00:11.425 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_27 /tmp/tmp.j44LZPowal/cluster2-mirror-image_27.status 2026-02-22T22:00:11.425 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:00:11.425 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:00:11.425 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_27 2026-02-22T22:00:11.425 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster2-mirror-image_27.status 2026-02-22T22:00:11.425 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_27 --format xml 2026-02-22T22:00:11.425 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:00:11.453 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T22:00:11.455 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snapshot_id=138 2026-02-22T22:00:11.455 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T22:00:11.455 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:00:11.455 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.2 2026-02-22T22:00:11.656 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_27 /tmp/tmp.j44LZPowal/cluster1-mirror-image_27.status 2026-02-22T22:00:11.656 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:00:11.657 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:00:11.657 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_27 2026-02-22T22:00:11.657 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_27.status 2026-02-22T22:00:11.657 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_27 --format xml 2026-02-22T22:00:11.657 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:00:11.689 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T22:00:11.689 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T22:00:11.690 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T22:00:11.690 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T22:00:11.690 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T22:00:11.691 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T22:00:11.691 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 138 = '' 2026-02-22T22:00:11.691 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:00:11.691 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.4 2026-02-22T22:00:12.092 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_27 /tmp/tmp.j44LZPowal/cluster1-mirror-image_27.status 2026-02-22T22:00:12.092 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:00:12.092 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:00:12.092 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_27 2026-02-22T22:00:12.092 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_27.status 2026-02-22T22:00:12.093 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_27 --format xml 2026-02-22T22:00:12.093 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:00:12.120 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T22:00:12.121 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T22:00:12.121 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T22:00:12.122 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T22:00:12.122 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T22:00:12.122 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T22:00:12.123 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 138 = '' 2026-02-22T22:00:12.123 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:00:12.123 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.8 2026-02-22T22:00:12.924 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_27 /tmp/tmp.j44LZPowal/cluster1-mirror-image_27.status 2026-02-22T22:00:12.924 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:00:12.924 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:00:12.924 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_27 2026-02-22T22:00:12.924 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_27.status 2026-02-22T22:00:12.924 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_27 --format xml 2026-02-22T22:00:12.925 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:00:12.954 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T22:00:12.956 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id=138 2026-02-22T22:00:12.956 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 138 = 138 2026-02-22T22:00:12.956 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:00:12.956 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snap_present cluster1 mirror image_27 snap 2026-02-22T22:00:12.956 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:00:12.956 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:00:12.956 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_27 2026-02-22T22:00:12.956 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T22:00:12.956 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:00:12.957 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T22:00:12.957 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T22:00:13.956 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 info mirror/image_27@snap 2026-02-22T22:00:13.983 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_27': 2026-02-22T22:00:13.984 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 128 MiB in 32 objects 2026-02-22T22:00:13.984 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T22:00:13.984 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 2 2026-02-22T22:00:13.984 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 1227de8453bf 2026-02-22T22:00:13.984 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.1227de8453bf 2026-02-22T22:00:13.984 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T22:00:13.984 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock, non-primary 2026-02-22T22:00:13.984 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T22:00:13.985 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T22:00:13.985 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:52:25 2026 2026-02-22T22:00:13.985 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:52:25 2026 2026-02-22T22:00:13.985 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:52:25 2026 2026-02-22T22:00:13.985 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: protected: False 2026-02-22T22:00:13.985 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T22:00:13.985 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T22:00:13.985 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: 226f7166-427f-4ef9-893a-6179cde62b6d 2026-02-22T22:00:13.985 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: false 2026-02-22T22:00:13.987 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:00:13.987 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ compare_image_snaps mirror image_27 snap 2026-02-22T22:00:13.987 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:00:13.987 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_27 2026-02-22T22:00:13.987 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T22:00:13.987 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local ret=0 2026-02-22T22:00:13.988 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local rmt_export=/tmp/tmp.j44LZPowal/cluster2-mirror-image_27.export 2026-02-22T22:00:13.988 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local loc_export=/tmp/tmp.j44LZPowal/cluster1-mirror-image_27.export 2026-02-22T22:00:13.988 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-image_27.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_27.export 2026-02-22T22:00:13.988 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror export image_27@snap /tmp/tmp.j44LZPowal/cluster2-mirror-image_27.export 2026-02-22T22:00:14.095 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:00:14.103 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 -p mirror export image_27@snap /tmp/tmp.j44LZPowal/cluster1-mirror-image_27.export 2026-02-22T22:00:14.229 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:00:14.238 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ cmp /tmp/tmp.j44LZPowal/cluster2-mirror-image_27.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_27.export 2026-02-22T22:00:14.286 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-image_27.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_27.export 2026-02-22T22:00:14.320 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:00:14.320 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:00:14.321 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_28 2026-02-22T22:00:14.321 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_snap cluster2 mirror image_28 snap 2026-02-22T22:00:14.321 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:00:14.321 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:00:14.321 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_28 2026-02-22T22:00:14.321 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T22:00:14.321 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror snap create image_28@snap --debug-rbd=20 --debug-journaler=20 2026-02-22T22:00:14.656 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.0 2026-02-22T22:00:14.656 INFO:teuthology.orchestra.run:waiting for 600 2026-02-22T22:00:14.657 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.0 2026-02-22T22:00:14.657 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Restarting daemon 2026-02-22T22:00:14.657 DEBUG:teuthology.orchestra.run.trial026:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 2026-02-22T22:00:14.659 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Started 2026-02-22T22:00:14.660 INFO:tasks.rbd_mirror_thrash:waiting for 89.12107987929126 secs before thrashing 2026-02-22T22:00:15.423 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_started cluster1 mirror image_28 2026-02-22T22:00:15.423 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:00:15.423 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:00:15.423 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_28 2026-02-22T22:00:15.423 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_state cluster1 mirror image_28 started 2026-02-22T22:00:15.423 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:00:15.423 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:00:15.424 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_28 2026-02-22T22:00:15.424 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=started 2026-02-22T22:00:15.424 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:00:15.424 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:00:15.424 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T22:00:16.424 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_replay_state cluster1 mirror image_28 started 2026-02-22T22:00:16.424 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:00:16.425 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:00:16.425 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_28 2026-02-22T22:00:16.425 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=started 2026-02-22T22:00:16.425 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_result 2026-02-22T22:00:16.425 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=stopped 2026-02-22T22:00:16.425 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_28 2026-02-22T22:00:16.425 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_instance=cluster1 2026-02-22T22:00:16.425 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:00:16.425 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:00:16.425 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance=cluster1 2026-02-22T22:00:16.426 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local loop_instance 2026-02-22T22:00:16.426 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:00:16.426 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sleep 0 2026-02-22T22:00:16.426 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ grep -i state 2026-02-22T22:00:16.429 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:00:16.429 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ seq 0 3 2026-02-22T22:00:16.429 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:00:16.429 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_28 2026-02-22T22:00:16.429 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:0 2026-02-22T22:00:16.429 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:00:16.430 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T22:00:16.430 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:00:16.430 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:0 2026-02-22T22:00:16.430 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T22:00:16.430 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T22:00:16.430 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:00:16.430 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=0 2026-02-22T22:00:16.430 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:00:16.430 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T22:00:16.430 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:00:16.430 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=0 2026-02-22T22:00:16.430 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T22:00:16.431 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:00:16.431 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:00:16.431 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T22:00:16.431 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T22:00:16.431 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:00:16.431 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:0 2026-02-22T22:00:16.431 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:00:16.431 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T22:00:16.431 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:00:16.431 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T22:00:16.431 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:00:16.431 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:00:16.432 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T22:00:16.432 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=0 2026-02-22T22:00:16.432 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=0 2026-02-22T22:00:16.432 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:00:16.440 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:00:16.440 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:00:16.440 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:00:16.440 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_28 2026-02-22T22:00:16.484 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_28'] 2026-02-22T22:00:16.485 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:00:16.485 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T22:00:16.487 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:00:16.487 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_28 2026-02-22T22:00:16.487 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:1 2026-02-22T22:00:16.487 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:00:16.487 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T22:00:16.488 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:00:16.488 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:1 2026-02-22T22:00:16.488 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T22:00:16.488 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T22:00:16.488 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:00:16.488 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=1 2026-02-22T22:00:16.488 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:00:16.488 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T22:00:16.488 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:00:16.488 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=1 2026-02-22T22:00:16.488 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T22:00:16.489 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:00:16.489 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:1 2026-02-22T22:00:16.489 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T22:00:16.489 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T22:00:16.489 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:00:16.489 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:1 2026-02-22T22:00:16.489 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:00:16.489 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T22:00:16.489 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:00:16.489 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T22:00:16.489 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T22:00:16.489 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:00:16.490 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T22:00:16.490 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=1 2026-02-22T22:00:16.490 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=1 2026-02-22T22:00:16.490 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:00:16.499 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:00:16.500 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:00:16.500 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:00:16.500 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_28 2026-02-22T22:00:16.544 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_28'] 2026-02-22T22:00:16.544 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:00:16.544 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T22:00:16.547 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:00:16.547 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_28 2026-02-22T22:00:16.547 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:2 2026-02-22T22:00:16.547 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:00:16.547 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T22:00:16.547 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:00:16.547 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:2 2026-02-22T22:00:16.548 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T22:00:16.548 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T22:00:16.548 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:00:16.548 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=2 2026-02-22T22:00:16.548 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:00:16.548 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T22:00:16.548 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:00:16.548 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=2 2026-02-22T22:00:16.548 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T22:00:16.548 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:00:16.548 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:2 2026-02-22T22:00:16.548 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T22:00:16.549 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T22:00:16.549 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:00:16.549 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:2 2026-02-22T22:00:16.549 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:00:16.549 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T22:00:16.549 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:00:16.549 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T22:00:16.549 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T22:00:16.549 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:00:16.549 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T22:00:16.549 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=2 2026-02-22T22:00:16.549 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=2 2026-02-22T22:00:16.549 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:00:16.557 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:00:16.558 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:00:16.558 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:00:16.558 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_28 2026-02-22T22:00:16.601 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_28'] 2026-02-22T22:00:16.601 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:00:16.601 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T22:00:16.604 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:00:16.604 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_28 2026-02-22T22:00:16.604 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:3 2026-02-22T22:00:16.605 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:00:16.605 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T22:00:16.605 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T22:00:16.605 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:3 2026-02-22T22:00:16.605 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T22:00:16.605 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T22:00:16.605 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:00:16.605 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=3 2026-02-22T22:00:16.605 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T22:00:16.605 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T22:00:16.605 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:00:16.605 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=3 2026-02-22T22:00:16.606 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=3 2026-02-22T22:00:16.606 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T22:00:16.606 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:3 2026-02-22T22:00:16.606 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T22:00:16.606 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T22:00:16.606 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T22:00:16.606 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:3 2026-02-22T22:00:16.606 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:00:16.606 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T22:00:16.606 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:00:16.606 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=3 2026-02-22T22:00:16.606 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T22:00:16.607 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:00:16.607 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T22:00:16.607 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=3 2026-02-22T22:00:16.607 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=3 2026-02-22T22:00:16.607 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T22:00:16.614 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:00:16.615 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:00:16.615 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:00:16.615 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_28 2026-02-22T22:00:16.659 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_28'] 2026-02-22T22:00:16.659 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:00:16.659 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T22:00:16.660 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:00:16.660 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sleep 1 2026-02-22T22:00:17.662 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:00:17.662 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ seq 0 3 2026-02-22T22:00:17.666 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:00:17.666 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_28 2026-02-22T22:00:17.666 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:0 2026-02-22T22:00:17.666 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:00:17.666 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T22:00:17.666 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:00:17.667 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:0 2026-02-22T22:00:17.667 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T22:00:17.667 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T22:00:17.667 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:00:17.667 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=0 2026-02-22T22:00:17.667 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:00:17.667 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T22:00:17.667 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:00:17.667 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=0 2026-02-22T22:00:17.667 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T22:00:17.667 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:00:17.667 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:00:17.668 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T22:00:17.668 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T22:00:17.668 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:00:17.668 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:0 2026-02-22T22:00:17.668 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:00:17.668 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T22:00:17.668 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:00:17.668 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T22:00:17.668 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:00:17.668 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:00:17.668 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T22:00:17.668 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=0 2026-02-22T22:00:17.669 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=0 2026-02-22T22:00:17.669 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:00:17.675 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:00:17.675 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:00:17.676 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:00:17.676 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_28 2026-02-22T22:00:17.720 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_28'] 2026-02-22T22:00:17.720 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:00:17.721 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T22:00:17.723 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:00:17.724 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_28 2026-02-22T22:00:17.724 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:1 2026-02-22T22:00:17.724 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:00:17.724 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T22:00:17.724 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:00:17.724 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:1 2026-02-22T22:00:17.724 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T22:00:17.724 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T22:00:17.724 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:00:17.724 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=1 2026-02-22T22:00:17.724 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:00:17.725 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T22:00:17.725 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:00:17.725 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=1 2026-02-22T22:00:17.725 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T22:00:17.725 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:00:17.725 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:1 2026-02-22T22:00:17.725 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T22:00:17.725 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T22:00:17.725 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:00:17.725 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:1 2026-02-22T22:00:17.725 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:00:17.725 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T22:00:17.726 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:00:17.726 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T22:00:17.726 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T22:00:17.726 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:00:17.726 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T22:00:17.726 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=1 2026-02-22T22:00:17.726 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=1 2026-02-22T22:00:17.726 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:00:17.733 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:00:17.734 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:00:17.734 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:00:17.734 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_28 2026-02-22T22:00:17.778 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_28'] 2026-02-22T22:00:17.778 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:00:17.778 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T22:00:17.781 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:00:17.781 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_28 2026-02-22T22:00:17.781 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:2 2026-02-22T22:00:17.781 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:00:17.781 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T22:00:17.781 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:00:17.781 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:2 2026-02-22T22:00:17.781 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T22:00:17.781 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T22:00:17.781 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:00:17.782 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=2 2026-02-22T22:00:17.782 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:00:17.782 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T22:00:17.782 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:00:17.782 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=2 2026-02-22T22:00:17.782 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T22:00:17.782 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:00:17.782 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:2 2026-02-22T22:00:17.782 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T22:00:17.782 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T22:00:17.782 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:00:17.783 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:2 2026-02-22T22:00:17.783 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:00:17.783 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T22:00:17.783 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:00:17.783 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T22:00:17.783 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T22:00:17.783 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:00:17.783 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T22:00:17.783 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=2 2026-02-22T22:00:17.783 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=2 2026-02-22T22:00:17.783 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:00:17.791 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:00:17.791 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:00:17.791 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:00:17.791 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_28 2026-02-22T22:00:17.836 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_28'] 2026-02-22T22:00:17.836 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:00:17.836 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T22:00:17.840 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:00:17.840 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_28 2026-02-22T22:00:17.840 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:3 2026-02-22T22:00:17.840 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:00:17.840 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T22:00:17.840 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T22:00:17.840 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:3 2026-02-22T22:00:17.840 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T22:00:17.840 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T22:00:17.840 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:00:17.841 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=3 2026-02-22T22:00:17.841 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T22:00:17.841 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T22:00:17.841 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:00:17.841 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=3 2026-02-22T22:00:17.841 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=3 2026-02-22T22:00:17.841 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T22:00:17.841 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:3 2026-02-22T22:00:17.841 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T22:00:17.841 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T22:00:17.841 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T22:00:17.841 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:3 2026-02-22T22:00:17.842 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:00:17.842 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T22:00:17.842 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:00:17.842 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=3 2026-02-22T22:00:17.842 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T22:00:17.842 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:00:17.842 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T22:00:17.842 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=3 2026-02-22T22:00:17.842 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=3 2026-02-22T22:00:17.842 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T22:00:17.849 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:00:17.850 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:00:17.850 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:00:17.850 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_28 2026-02-22T22:00:17.894 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_28'] 2026-02-22T22:00:17.894 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:00:17.895 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T22:00:17.895 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:00:17.896 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sleep 2 2026-02-22T22:00:19.897 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:00:19.897 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ seq 0 3 2026-02-22T22:00:19.900 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:00:19.901 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_28 2026-02-22T22:00:19.901 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:0 2026-02-22T22:00:19.901 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:00:19.901 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T22:00:19.901 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:00:19.901 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:0 2026-02-22T22:00:19.901 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T22:00:19.901 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T22:00:19.901 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:00:19.901 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=0 2026-02-22T22:00:19.902 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:00:19.902 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T22:00:19.902 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:00:19.902 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=0 2026-02-22T22:00:19.902 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T22:00:19.902 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:00:19.902 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:00:19.902 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T22:00:19.902 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T22:00:19.902 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:00:19.902 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:0 2026-02-22T22:00:19.902 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:00:19.903 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T22:00:19.903 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:00:19.903 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T22:00:19.903 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:00:19.903 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:00:19.903 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T22:00:19.903 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=0 2026-02-22T22:00:19.903 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=0 2026-02-22T22:00:19.903 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:00:19.911 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:00:19.911 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:00:19.911 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:00:19.912 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_28 2026-02-22T22:00:19.956 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_28'] 2026-02-22T22:00:19.957 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:00:19.957 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T22:00:19.960 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:00:19.960 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_28 2026-02-22T22:00:19.960 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:1 2026-02-22T22:00:19.960 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:00:19.960 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T22:00:19.960 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:00:19.960 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:1 2026-02-22T22:00:19.960 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T22:00:19.960 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T22:00:19.961 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:00:19.961 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=1 2026-02-22T22:00:19.961 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:00:19.961 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T22:00:19.961 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:00:19.961 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=1 2026-02-22T22:00:19.961 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T22:00:19.961 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:00:19.961 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:1 2026-02-22T22:00:19.961 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T22:00:19.962 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T22:00:19.962 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:00:19.962 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:1 2026-02-22T22:00:19.962 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:00:19.962 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T22:00:19.962 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:00:19.962 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T22:00:19.962 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T22:00:19.962 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:00:19.962 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T22:00:19.962 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=1 2026-02-22T22:00:19.962 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=1 2026-02-22T22:00:19.963 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:00:19.970 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:00:19.970 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:00:19.970 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:00:19.970 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_28 2026-02-22T22:00:20.014 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_28'] 2026-02-22T22:00:20.014 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:00:20.015 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T22:00:20.018 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:00:20.018 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_28 2026-02-22T22:00:20.018 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:2 2026-02-22T22:00:20.018 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:00:20.018 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T22:00:20.019 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:00:20.019 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:2 2026-02-22T22:00:20.019 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T22:00:20.019 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T22:00:20.019 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:00:20.019 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=2 2026-02-22T22:00:20.019 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:00:20.019 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T22:00:20.019 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:00:20.019 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=2 2026-02-22T22:00:20.019 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T22:00:20.019 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:00:20.020 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:2 2026-02-22T22:00:20.020 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T22:00:20.020 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T22:00:20.020 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:00:20.020 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:2 2026-02-22T22:00:20.020 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:00:20.020 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T22:00:20.020 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:00:20.020 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T22:00:20.020 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T22:00:20.020 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:00:20.020 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T22:00:20.020 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=2 2026-02-22T22:00:20.021 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=2 2026-02-22T22:00:20.021 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:00:20.028 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:00:20.028 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:00:20.028 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:00:20.028 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_28 2026-02-22T22:00:20.072 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_28'] 2026-02-22T22:00:20.072 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:00:20.073 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T22:00:20.075 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:00:20.076 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_28 2026-02-22T22:00:20.076 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:3 2026-02-22T22:00:20.076 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:00:20.076 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T22:00:20.076 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T22:00:20.076 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:3 2026-02-22T22:00:20.076 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T22:00:20.076 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T22:00:20.076 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:00:20.076 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=3 2026-02-22T22:00:20.077 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T22:00:20.077 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T22:00:20.077 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:00:20.077 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=3 2026-02-22T22:00:20.077 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=3 2026-02-22T22:00:20.077 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T22:00:20.077 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:3 2026-02-22T22:00:20.077 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T22:00:20.077 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T22:00:20.077 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T22:00:20.077 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:3 2026-02-22T22:00:20.077 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:00:20.078 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T22:00:20.078 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:00:20.078 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=3 2026-02-22T22:00:20.078 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T22:00:20.078 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:00:20.078 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T22:00:20.078 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=3 2026-02-22T22:00:20.078 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=3 2026-02-22T22:00:20.078 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T22:00:20.085 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:00:20.086 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:00:20.086 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:00:20.086 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_28 2026-02-22T22:00:20.129 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_28'] 2026-02-22T22:00:20.129 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:00:20.129 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T22:00:20.130 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:00:20.130 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sleep 4 2026-02-22T22:00:24.131 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:00:24.132 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ seq 0 3 2026-02-22T22:00:24.135 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:00:24.135 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_28 2026-02-22T22:00:24.135 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:0 2026-02-22T22:00:24.135 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:00:24.135 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T22:00:24.135 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:00:24.135 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:0 2026-02-22T22:00:24.135 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T22:00:24.136 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T22:00:24.136 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:00:24.136 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=0 2026-02-22T22:00:24.136 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:00:24.136 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T22:00:24.136 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:00:24.136 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=0 2026-02-22T22:00:24.136 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T22:00:24.136 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:00:24.136 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:00:24.136 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T22:00:24.136 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T22:00:24.137 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:00:24.137 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:0 2026-02-22T22:00:24.137 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:00:24.137 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T22:00:24.137 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:00:24.137 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T22:00:24.137 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:00:24.137 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:00:24.137 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T22:00:24.137 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=0 2026-02-22T22:00:24.137 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=0 2026-02-22T22:00:24.137 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:00:24.147 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:00:24.147 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:00:24.147 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:00:24.147 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_28 2026-02-22T22:00:24.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_28'] 2026-02-22T22:00:24.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:00:24.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T22:00:24.195 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:00:24.196 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_28 2026-02-22T22:00:24.196 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:1 2026-02-22T22:00:24.196 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:00:24.196 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T22:00:24.196 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:00:24.196 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:1 2026-02-22T22:00:24.196 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T22:00:24.196 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T22:00:24.196 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:00:24.196 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=1 2026-02-22T22:00:24.196 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:00:24.196 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T22:00:24.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:00:24.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=1 2026-02-22T22:00:24.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T22:00:24.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:00:24.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:1 2026-02-22T22:00:24.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T22:00:24.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T22:00:24.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:00:24.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:1 2026-02-22T22:00:24.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:00:24.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T22:00:24.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:00:24.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T22:00:24.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T22:00:24.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:00:24.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T22:00:24.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=1 2026-02-22T22:00:24.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=1 2026-02-22T22:00:24.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:00:24.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:00:24.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:00:24.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:00:24.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_28 2026-02-22T22:00:24.249 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_28'] 2026-02-22T22:00:24.249 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:00:24.249 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T22:00:24.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:00:24.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_28 2026-02-22T22:00:24.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:2 2026-02-22T22:00:24.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:00:24.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T22:00:24.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:00:24.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:2 2026-02-22T22:00:24.254 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T22:00:24.254 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T22:00:24.254 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:00:24.254 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=2 2026-02-22T22:00:24.254 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:00:24.254 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T22:00:24.254 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:00:24.254 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=2 2026-02-22T22:00:24.254 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T22:00:24.254 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:00:24.254 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:2 2026-02-22T22:00:24.254 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T22:00:24.255 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T22:00:24.255 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:00:24.255 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:2 2026-02-22T22:00:24.255 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:00:24.255 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T22:00:24.255 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:00:24.255 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T22:00:24.255 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T22:00:24.255 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:00:24.255 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T22:00:24.255 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=2 2026-02-22T22:00:24.255 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=2 2026-02-22T22:00:24.256 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:00:24.263 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:00:24.263 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:00:24.263 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:00:24.264 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_28 2026-02-22T22:00:24.307 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_28'] 2026-02-22T22:00:24.307 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:00:24.308 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T22:00:24.311 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:00:24.311 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_28 2026-02-22T22:00:24.311 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:3 2026-02-22T22:00:24.311 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:00:24.311 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T22:00:24.311 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T22:00:24.311 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:3 2026-02-22T22:00:24.311 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T22:00:24.311 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T22:00:24.311 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:00:24.311 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=3 2026-02-22T22:00:24.312 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T22:00:24.312 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T22:00:24.312 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:00:24.312 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=3 2026-02-22T22:00:24.312 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=3 2026-02-22T22:00:24.312 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T22:00:24.312 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:3 2026-02-22T22:00:24.312 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T22:00:24.312 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T22:00:24.312 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T22:00:24.312 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:3 2026-02-22T22:00:24.312 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:00:24.313 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T22:00:24.313 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:00:24.313 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=3 2026-02-22T22:00:24.313 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T22:00:24.313 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:00:24.313 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T22:00:24.313 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=3 2026-02-22T22:00:24.313 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=3 2026-02-22T22:00:24.313 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T22:00:24.320 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:00:24.321 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:00:24.321 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:00:24.321 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_28 2026-02-22T22:00:24.364 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_28'] 2026-02-22T22:00:24.364 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:00:24.365 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T22:00:24.365 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:00:24.366 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sleep 8 2026-02-22T22:00:32.367 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:00:32.367 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ seq 0 3 2026-02-22T22:00:32.370 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:00:32.370 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_28 2026-02-22T22:00:32.370 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:0 2026-02-22T22:00:32.370 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:00:32.370 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T22:00:32.370 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:00:32.370 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:0 2026-02-22T22:00:32.370 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T22:00:32.371 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T22:00:32.371 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:00:32.371 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=0 2026-02-22T22:00:32.371 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:00:32.371 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T22:00:32.371 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:00:32.371 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=0 2026-02-22T22:00:32.371 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T22:00:32.371 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:00:32.371 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:00:32.371 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T22:00:32.371 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T22:00:32.371 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:00:32.371 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:0 2026-02-22T22:00:32.371 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:00:32.372 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T22:00:32.372 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:00:32.372 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T22:00:32.372 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:00:32.372 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:00:32.372 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T22:00:32.372 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=0 2026-02-22T22:00:32.372 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=0 2026-02-22T22:00:32.372 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:00:32.381 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:00:32.381 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:00:32.381 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:00:32.381 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_28 2026-02-22T22:00:32.425 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_28'] 2026-02-22T22:00:32.426 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:00:32.426 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T22:00:32.429 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:00:32.429 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_28 2026-02-22T22:00:32.429 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:1 2026-02-22T22:00:32.429 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:00:32.429 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T22:00:32.429 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:00:32.429 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:1 2026-02-22T22:00:32.429 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T22:00:32.429 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T22:00:32.429 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:00:32.430 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=1 2026-02-22T22:00:32.430 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:00:32.430 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T22:00:32.430 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:00:32.430 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=1 2026-02-22T22:00:32.430 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T22:00:32.430 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:00:32.430 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:1 2026-02-22T22:00:32.430 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T22:00:32.430 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T22:00:32.430 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:00:32.430 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:1 2026-02-22T22:00:32.431 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:00:32.431 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T22:00:32.431 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:00:32.431 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T22:00:32.431 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T22:00:32.431 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:00:32.431 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T22:00:32.431 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=1 2026-02-22T22:00:32.431 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=1 2026-02-22T22:00:32.431 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:00:32.439 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:00:32.439 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:00:32.440 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:00:32.440 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_28 2026-02-22T22:00:32.482 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_28'] 2026-02-22T22:00:32.482 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:00:32.482 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T22:00:32.486 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:00:32.486 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_28 2026-02-22T22:00:32.486 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:2 2026-02-22T22:00:32.486 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:00:32.486 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T22:00:32.486 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:00:32.486 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:2 2026-02-22T22:00:32.487 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T22:00:32.487 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T22:00:32.487 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:00:32.487 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=2 2026-02-22T22:00:32.487 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:00:32.487 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T22:00:32.487 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:00:32.487 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=2 2026-02-22T22:00:32.487 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T22:00:32.487 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:00:32.487 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:2 2026-02-22T22:00:32.487 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T22:00:32.488 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T22:00:32.488 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:00:32.488 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:2 2026-02-22T22:00:32.488 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:00:32.488 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T22:00:32.488 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:00:32.488 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T22:00:32.488 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T22:00:32.488 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:00:32.488 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T22:00:32.488 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=2 2026-02-22T22:00:32.489 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=2 2026-02-22T22:00:32.489 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:00:32.497 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:00:32.497 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:00:32.497 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:00:32.497 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_28 2026-02-22T22:00:32.540 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_28'] 2026-02-22T22:00:32.540 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:00:32.540 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T22:00:32.543 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:00:32.543 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_28 2026-02-22T22:00:32.544 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:3 2026-02-22T22:00:32.544 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:00:32.544 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T22:00:32.544 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T22:00:32.544 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:3 2026-02-22T22:00:32.544 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T22:00:32.544 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T22:00:32.544 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:00:32.544 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=3 2026-02-22T22:00:32.544 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T22:00:32.544 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T22:00:32.545 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:00:32.545 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=3 2026-02-22T22:00:32.545 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=3 2026-02-22T22:00:32.545 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T22:00:32.545 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:3 2026-02-22T22:00:32.545 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T22:00:32.545 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T22:00:32.545 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T22:00:32.545 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:3 2026-02-22T22:00:32.545 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:00:32.545 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T22:00:32.545 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:00:32.545 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=3 2026-02-22T22:00:32.546 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T22:00:32.546 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:00:32.546 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T22:00:32.546 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=3 2026-02-22T22:00:32.546 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=3 2026-02-22T22:00:32.546 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T22:00:32.554 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:00:32.555 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:00:32.555 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:00:32.555 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_28 2026-02-22T22:00:32.598 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_28'] 2026-02-22T22:00:32.598 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:00:32.598 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T22:00:32.599 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:00:32.599 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sleep 8 2026-02-22T22:00:40.600 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:00:40.601 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ seq 0 3 2026-02-22T22:00:40.604 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:00:40.604 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_28 2026-02-22T22:00:40.604 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:0 2026-02-22T22:00:40.604 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:00:40.604 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T22:00:40.604 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:00:40.604 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:0 2026-02-22T22:00:40.604 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T22:00:40.604 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T22:00:40.604 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:00:40.605 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=0 2026-02-22T22:00:40.605 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:00:40.605 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T22:00:40.605 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:00:40.605 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=0 2026-02-22T22:00:40.605 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T22:00:40.605 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:00:40.605 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:00:40.605 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T22:00:40.605 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T22:00:40.606 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:00:40.606 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:0 2026-02-22T22:00:40.606 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:00:40.606 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T22:00:40.606 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:00:40.606 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T22:00:40.606 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:00:40.606 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:00:40.606 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T22:00:40.606 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=0 2026-02-22T22:00:40.606 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=0 2026-02-22T22:00:40.606 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:00:40.615 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:00:40.615 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:00:40.615 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:00:40.615 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_28 2026-02-22T22:00:40.660 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_28'] 2026-02-22T22:00:40.660 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:00:40.660 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T22:00:40.663 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:00:40.663 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_28 2026-02-22T22:00:40.663 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:1 2026-02-22T22:00:40.664 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:00:40.664 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T22:00:40.664 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:00:40.664 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:1 2026-02-22T22:00:40.664 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T22:00:40.664 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T22:00:40.664 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:00:40.664 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=1 2026-02-22T22:00:40.664 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:00:40.664 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T22:00:40.664 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:00:40.665 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=1 2026-02-22T22:00:40.665 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T22:00:40.665 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:00:40.665 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:1 2026-02-22T22:00:40.665 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T22:00:40.665 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T22:00:40.665 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:00:40.665 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:1 2026-02-22T22:00:40.665 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:00:40.665 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T22:00:40.665 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:00:40.665 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T22:00:40.666 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T22:00:40.666 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:00:40.666 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T22:00:40.666 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=1 2026-02-22T22:00:40.666 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=1 2026-02-22T22:00:40.666 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:00:40.673 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:00:40.674 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:00:40.674 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:00:40.674 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_28 2026-02-22T22:00:40.718 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_28'] 2026-02-22T22:00:40.718 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:00:40.718 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T22:00:40.721 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:00:40.721 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_28 2026-02-22T22:00:40.721 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:2 2026-02-22T22:00:40.721 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:00:40.721 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T22:00:40.721 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:00:40.721 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:2 2026-02-22T22:00:40.721 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T22:00:40.721 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T22:00:40.721 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:00:40.722 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=2 2026-02-22T22:00:40.722 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:00:40.722 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T22:00:40.722 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:00:40.722 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=2 2026-02-22T22:00:40.722 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T22:00:40.722 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:00:40.722 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:2 2026-02-22T22:00:40.722 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T22:00:40.722 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T22:00:40.722 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:00:40.722 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:2 2026-02-22T22:00:40.722 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:00:40.723 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T22:00:40.723 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:00:40.723 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T22:00:40.723 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T22:00:40.723 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:00:40.723 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T22:00:40.723 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=2 2026-02-22T22:00:40.723 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=2 2026-02-22T22:00:40.723 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:00:40.731 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:00:40.731 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:00:40.731 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:00:40.731 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_28 2026-02-22T22:00:40.776 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_28'] 2026-02-22T22:00:40.776 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:00:40.776 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T22:00:40.779 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:00:40.779 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_28 2026-02-22T22:00:40.779 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:3 2026-02-22T22:00:40.779 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:00:40.779 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T22:00:40.780 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T22:00:40.780 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:3 2026-02-22T22:00:40.780 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T22:00:40.780 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T22:00:40.780 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:00:40.780 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=3 2026-02-22T22:00:40.780 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T22:00:40.780 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T22:00:40.780 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:00:40.780 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=3 2026-02-22T22:00:40.780 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=3 2026-02-22T22:00:40.780 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T22:00:40.781 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:3 2026-02-22T22:00:40.781 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T22:00:40.781 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T22:00:40.781 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T22:00:40.781 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:3 2026-02-22T22:00:40.781 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:00:40.781 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T22:00:40.781 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:00:40.781 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=3 2026-02-22T22:00:40.781 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T22:00:40.781 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:00:40.781 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T22:00:40.782 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=3 2026-02-22T22:00:40.782 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=3 2026-02-22T22:00:40.782 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T22:00:40.789 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:00:40.789 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:00:40.789 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:00:40.790 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_28 2026-02-22T22:00:40.839 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ return 0 2026-02-22T22:00:40.839 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:00:40.840 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:00:40.840 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep -i Replaying 2026-02-22T22:00:40.840 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: "state": "Replaying" 2026-02-22T22:00:40.841 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_state=started 2026-02-22T22:00:40.842 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test started = started 2026-02-22T22:00:40.842 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:00:40.842 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_28 2026-02-22T22:00:40.842 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T22:00:40.842 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:00:40.842 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:00:40.842 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_28 2026-02-22T22:00:40.842 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = journal ']' 2026-02-22T22:00:40.842 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T22:00:40.842 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ mirror_image_snapshot cluster2 mirror image_28 2026-02-22T22:00:40.842 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:00:40.843 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:00:40.843 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_28 2026-02-22T22:00:40.843 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_28 2026-02-22T22:00:41.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Snapshot ID: 140 2026-02-22T22:00:41.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_28 2026-02-22T22:00:41.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T22:00:41.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:00:41.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:00:41.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_28 2026-02-22T22:00:41.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster2-mirror-image_28.status 2026-02-22T22:00:41.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster2-mirror-image_28.status 2026-02-22T22:00:41.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:00:41.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_log=/tmp/tmp.j44LZPowal/cluster2-mirror-image_28.status 2026-02-22T22:00:41.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster1-mirror-image_28.status 2026-02-22T22:00:41.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster1-mirror-image_28.status 2026-02-22T22:00:41.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:00:41.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_status_log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_28.status 2026-02-22T22:00:41.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_28 /tmp/tmp.j44LZPowal/cluster2-mirror-image_28.status 2026-02-22T22:00:41.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:00:41.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:00:41.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_28 2026-02-22T22:00:41.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster2-mirror-image_28.status 2026-02-22T22:00:41.203 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_28 --format xml 2026-02-22T22:00:41.203 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:00:41.230 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T22:00:41.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snapshot_id=140 2026-02-22T22:00:41.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T22:00:41.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:00:41.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.2 2026-02-22T22:00:41.433 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_28 /tmp/tmp.j44LZPowal/cluster1-mirror-image_28.status 2026-02-22T22:00:41.433 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:00:41.434 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:00:41.434 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_28 2026-02-22T22:00:41.434 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_28.status 2026-02-22T22:00:41.434 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_28 --format xml 2026-02-22T22:00:41.434 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:00:41.465 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T22:00:41.465 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T22:00:41.466 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T22:00:41.466 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T22:00:41.466 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T22:00:41.467 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T22:00:41.467 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 140 = '' 2026-02-22T22:00:41.467 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:00:41.467 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.4 2026-02-22T22:00:41.868 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_28 /tmp/tmp.j44LZPowal/cluster1-mirror-image_28.status 2026-02-22T22:00:41.868 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:00:41.868 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:00:41.868 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_28 2026-02-22T22:00:41.868 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_28.status 2026-02-22T22:00:41.868 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_28 --format xml 2026-02-22T22:00:41.869 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:00:41.898 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T22:00:41.898 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T22:00:41.899 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T22:00:41.899 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T22:00:41.899 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T22:00:41.900 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T22:00:41.900 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 140 = '' 2026-02-22T22:00:41.900 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:00:41.900 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.8 2026-02-22T22:00:42.701 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_28 /tmp/tmp.j44LZPowal/cluster1-mirror-image_28.status 2026-02-22T22:00:42.701 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:00:42.701 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:00:42.701 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_28 2026-02-22T22:00:42.701 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_28.status 2026-02-22T22:00:42.701 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_28 --format xml 2026-02-22T22:00:42.702 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:00:42.734 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T22:00:42.735 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id=140 2026-02-22T22:00:42.736 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 140 = 140 2026-02-22T22:00:42.736 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:00:42.736 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snap_present cluster1 mirror image_28 snap 2026-02-22T22:00:42.736 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:00:42.736 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:00:42.736 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_28 2026-02-22T22:00:42.736 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T22:00:42.736 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:00:42.736 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T22:00:42.736 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T22:00:43.736 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 info mirror/image_28@snap 2026-02-22T22:00:43.766 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_28': 2026-02-22T22:00:43.766 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 128 MiB in 32 objects 2026-02-22T22:00:43.766 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T22:00:43.766 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 2 2026-02-22T22:00:43.766 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 12481f41ff5a 2026-02-22T22:00:43.766 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.12481f41ff5a 2026-02-22T22:00:43.766 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T22:00:43.767 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock, non-primary 2026-02-22T22:00:43.767 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T22:00:43.767 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T22:00:43.767 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:52:38 2026 2026-02-22T22:00:43.767 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:52:38 2026 2026-02-22T22:00:43.767 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:52:38 2026 2026-02-22T22:00:43.767 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: protected: False 2026-02-22T22:00:43.767 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T22:00:43.767 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T22:00:43.767 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: 8b05d225-eedc-4b0a-94f7-6d3c0ca41d35 2026-02-22T22:00:43.767 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: false 2026-02-22T22:00:43.769 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:00:43.769 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ compare_image_snaps mirror image_28 snap 2026-02-22T22:00:43.769 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:00:43.769 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_28 2026-02-22T22:00:43.769 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T22:00:43.769 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local ret=0 2026-02-22T22:00:43.769 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local rmt_export=/tmp/tmp.j44LZPowal/cluster2-mirror-image_28.export 2026-02-22T22:00:43.769 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local loc_export=/tmp/tmp.j44LZPowal/cluster1-mirror-image_28.export 2026-02-22T22:00:43.769 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-image_28.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_28.export 2026-02-22T22:00:43.770 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror export image_28@snap /tmp/tmp.j44LZPowal/cluster2-mirror-image_28.export 2026-02-22T22:00:43.878 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:00:43.886 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 -p mirror export image_28@snap /tmp/tmp.j44LZPowal/cluster1-mirror-image_28.export 2026-02-22T22:00:44.004 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:00:44.011 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ cmp /tmp/tmp.j44LZPowal/cluster2-mirror-image_28.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_28.export 2026-02-22T22:00:44.062 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-image_28.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_28.export 2026-02-22T22:00:44.097 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:00:44.097 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:00:44.097 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_29 2026-02-22T22:00:44.098 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_snap cluster2 mirror image_29 snap 2026-02-22T22:00:44.098 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:00:44.098 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:00:44.098 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_29 2026-02-22T22:00:44.098 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T22:00:44.098 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror snap create image_29@snap --debug-rbd=20 --debug-journaler=20 2026-02-22T22:00:45.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_started cluster1 mirror image_29 2026-02-22T22:00:45.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:00:45.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:00:45.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_29 2026-02-22T22:00:45.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_state cluster1 mirror image_29 started 2026-02-22T22:00:45.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:00:45.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:00:45.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_29 2026-02-22T22:00:45.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=started 2026-02-22T22:00:45.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:00:45.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:00:45.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T22:00:46.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_replay_state cluster1 mirror image_29 started 2026-02-22T22:00:46.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:00:46.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:00:46.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_29 2026-02-22T22:00:46.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=started 2026-02-22T22:00:46.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_result 2026-02-22T22:00:46.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=stopped 2026-02-22T22:00:46.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_29 2026-02-22T22:00:46.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_instance=cluster1 2026-02-22T22:00:46.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:00:46.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:00:46.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ grep -i state 2026-02-22T22:00:46.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance=cluster1 2026-02-22T22:00:46.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local loop_instance 2026-02-22T22:00:46.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:00:46.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sleep 0 2026-02-22T22:00:46.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:00:46.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ seq 0 3 2026-02-22T22:00:46.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:00:46.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_29 2026-02-22T22:00:46.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:0 2026-02-22T22:00:46.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:00:46.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T22:00:46.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:00:46.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:0 2026-02-22T22:00:46.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T22:00:46.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T22:00:46.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:00:46.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=0 2026-02-22T22:00:46.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:00:46.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T22:00:46.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:00:46.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=0 2026-02-22T22:00:46.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T22:00:46.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:00:46.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:00:46.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T22:00:46.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T22:00:46.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:00:46.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:0 2026-02-22T22:00:46.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:00:46.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T22:00:46.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:00:46.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T22:00:46.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:00:46.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:00:46.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T22:00:46.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=0 2026-02-22T22:00:46.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=0 2026-02-22T22:00:46.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:00:46.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:00:46.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:00:46.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:00:46.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_29 2026-02-22T22:00:46.269 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ return 0 2026-02-22T22:00:46.269 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:00:46.270 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:00:46.270 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep -i Replaying 2026-02-22T22:00:46.270 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: "state": "Replaying" 2026-02-22T22:00:46.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_state=started 2026-02-22T22:00:46.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test started = started 2026-02-22T22:00:46.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:00:46.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_29 2026-02-22T22:00:46.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T22:00:46.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:00:46.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:00:46.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_29 2026-02-22T22:00:46.273 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = journal ']' 2026-02-22T22:00:46.273 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T22:00:46.273 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ mirror_image_snapshot cluster2 mirror image_29 2026-02-22T22:00:46.273 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:00:46.273 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:00:46.273 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_29 2026-02-22T22:00:46.273 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_29 2026-02-22T22:00:47.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Snapshot ID: 142 2026-02-22T22:00:47.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_29 2026-02-22T22:00:47.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T22:00:47.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:00:47.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:00:47.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_29 2026-02-22T22:00:47.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster2-mirror-image_29.status 2026-02-22T22:00:47.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster2-mirror-image_29.status 2026-02-22T22:00:47.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:00:47.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_log=/tmp/tmp.j44LZPowal/cluster2-mirror-image_29.status 2026-02-22T22:00:47.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster1-mirror-image_29.status 2026-02-22T22:00:47.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster1-mirror-image_29.status 2026-02-22T22:00:47.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:00:47.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_status_log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_29.status 2026-02-22T22:00:47.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_29 /tmp/tmp.j44LZPowal/cluster2-mirror-image_29.status 2026-02-22T22:00:47.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:00:47.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:00:47.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_29 2026-02-22T22:00:47.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster2-mirror-image_29.status 2026-02-22T22:00:47.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_29 --format xml 2026-02-22T22:00:47.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:00:47.237 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T22:00:47.239 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snapshot_id=142 2026-02-22T22:00:47.239 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T22:00:47.239 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:00:47.239 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.2 2026-02-22T22:00:47.440 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_29 /tmp/tmp.j44LZPowal/cluster1-mirror-image_29.status 2026-02-22T22:00:47.441 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:00:47.441 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:00:47.441 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_29 2026-02-22T22:00:47.441 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_29.status 2026-02-22T22:00:47.441 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_29 --format xml 2026-02-22T22:00:47.441 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:00:47.467 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T22:00:47.468 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T22:00:47.468 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T22:00:47.468 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T22:00:47.469 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T22:00:47.469 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T22:00:47.470 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 142 = '' 2026-02-22T22:00:47.470 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:00:47.470 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.4 2026-02-22T22:00:47.870 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_29 /tmp/tmp.j44LZPowal/cluster1-mirror-image_29.status 2026-02-22T22:00:47.871 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:00:47.871 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:00:47.871 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_29 2026-02-22T22:00:47.871 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_29.status 2026-02-22T22:00:47.872 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_29 --format xml 2026-02-22T22:00:47.872 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:00:47.898 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T22:00:47.898 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T22:00:47.899 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T22:00:47.899 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T22:00:47.899 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T22:00:47.900 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T22:00:47.900 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 142 = '' 2026-02-22T22:00:47.900 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:00:47.900 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.8 2026-02-22T22:00:48.701 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_29 /tmp/tmp.j44LZPowal/cluster1-mirror-image_29.status 2026-02-22T22:00:48.702 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:00:48.702 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:00:48.702 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_29 2026-02-22T22:00:48.702 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_29.status 2026-02-22T22:00:48.703 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_29 --format xml 2026-02-22T22:00:48.703 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:00:48.932 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T22:00:48.934 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id=142 2026-02-22T22:00:48.934 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 142 = 142 2026-02-22T22:00:48.935 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:00:48.935 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snap_present cluster1 mirror image_29 snap 2026-02-22T22:00:48.935 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:00:48.935 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:00:48.935 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_29 2026-02-22T22:00:48.935 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T22:00:48.935 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:00:48.935 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T22:00:48.935 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T22:00:49.935 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 info mirror/image_29@snap 2026-02-22T22:00:49.961 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_29': 2026-02-22T22:00:49.961 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 128 MiB in 32 objects 2026-02-22T22:00:49.961 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T22:00:49.961 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 2 2026-02-22T22:00:49.961 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 119a73f90507 2026-02-22T22:00:49.961 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.119a73f90507 2026-02-22T22:00:49.961 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T22:00:49.961 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock, non-primary 2026-02-22T22:00:49.961 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T22:00:49.961 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T22:00:49.961 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:52:45 2026 2026-02-22T22:00:49.961 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:52:45 2026 2026-02-22T22:00:49.961 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:52:45 2026 2026-02-22T22:00:49.961 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: protected: False 2026-02-22T22:00:49.961 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T22:00:49.961 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T22:00:49.961 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: 199d7709-f322-4604-895e-a583a38cb8d5 2026-02-22T22:00:49.961 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: false 2026-02-22T22:00:49.962 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:00:49.962 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ compare_image_snaps mirror image_29 snap 2026-02-22T22:00:49.962 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:00:49.962 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_29 2026-02-22T22:00:49.962 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T22:00:49.962 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local ret=0 2026-02-22T22:00:49.962 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local rmt_export=/tmp/tmp.j44LZPowal/cluster2-mirror-image_29.export 2026-02-22T22:00:49.963 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local loc_export=/tmp/tmp.j44LZPowal/cluster1-mirror-image_29.export 2026-02-22T22:00:49.963 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-image_29.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_29.export 2026-02-22T22:00:49.963 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror export image_29@snap /tmp/tmp.j44LZPowal/cluster2-mirror-image_29.export 2026-02-22T22:00:50.083 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:00:50.092 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 -p mirror export image_29@snap /tmp/tmp.j44LZPowal/cluster1-mirror-image_29.export 2026-02-22T22:00:50.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:00:50.223 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ cmp /tmp/tmp.j44LZPowal/cluster2-mirror-image_29.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_29.export 2026-02-22T22:00:50.274 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-image_29.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_29.export 2026-02-22T22:00:50.310 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:00:50.310 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:00:50.310 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_30 2026-02-22T22:00:50.310 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_snap cluster2 mirror image_30 snap 2026-02-22T22:00:50.310 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:00:50.310 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:00:50.311 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_30 2026-02-22T22:00:50.311 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T22:00:50.311 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror snap create image_30@snap --debug-rbd=20 --debug-journaler=20 2026-02-22T22:00:52.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_started cluster1 mirror image_30 2026-02-22T22:00:52.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:00:52.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:00:52.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_30 2026-02-22T22:00:52.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_state cluster1 mirror image_30 started 2026-02-22T22:00:52.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:00:52.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:00:52.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_30 2026-02-22T22:00:52.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=started 2026-02-22T22:00:52.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:00:52.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:00:52.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T22:00:53.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_replay_state cluster1 mirror image_30 started 2026-02-22T22:00:53.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:00:53.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:00:53.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_30 2026-02-22T22:00:53.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=started 2026-02-22T22:00:53.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_result 2026-02-22T22:00:53.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=stopped 2026-02-22T22:00:53.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_30 2026-02-22T22:00:53.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_instance=cluster1 2026-02-22T22:00:53.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:00:53.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:00:53.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance=cluster1 2026-02-22T22:00:53.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local loop_instance 2026-02-22T22:00:53.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ grep -i state 2026-02-22T22:00:53.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:00:53.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sleep 0 2026-02-22T22:00:53.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:00:53.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ seq 0 3 2026-02-22T22:00:53.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:00:53.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_30 2026-02-22T22:00:53.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:0 2026-02-22T22:00:53.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:00:53.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T22:00:53.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:00:53.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:0 2026-02-22T22:00:53.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T22:00:53.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T22:00:53.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:00:53.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=0 2026-02-22T22:00:53.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:00:53.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T22:00:53.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:00:53.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=0 2026-02-22T22:00:53.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T22:00:53.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:00:53.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:00:53.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T22:00:53.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T22:00:53.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:00:53.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:0 2026-02-22T22:00:53.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:00:53.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T22:00:53.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:00:53.213 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T22:00:53.213 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:00:53.213 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:00:53.213 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T22:00:53.213 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=0 2026-02-22T22:00:53.213 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=0 2026-02-22T22:00:53.213 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:00:53.220 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:00:53.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:00:53.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:00:53.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_30 2026-02-22T22:00:53.266 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2026-02-22T22:00:53.267 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:00:53.267 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T22:00:53.269 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:00:53.269 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_30 2026-02-22T22:00:53.269 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:1 2026-02-22T22:00:53.270 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:00:53.270 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T22:00:53.270 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:00:53.270 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:1 2026-02-22T22:00:53.270 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T22:00:53.270 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T22:00:53.270 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:00:53.270 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=1 2026-02-22T22:00:53.270 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:00:53.270 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T22:00:53.270 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:00:53.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=1 2026-02-22T22:00:53.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T22:00:53.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:00:53.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:1 2026-02-22T22:00:53.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T22:00:53.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T22:00:53.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:00:53.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:1 2026-02-22T22:00:53.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:00:53.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T22:00:53.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:00:53.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T22:00:53.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T22:00:53.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:00:53.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T22:00:53.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=1 2026-02-22T22:00:53.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=1 2026-02-22T22:00:53.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:00:53.279 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:00:53.279 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:00:53.280 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:00:53.280 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_30 2026-02-22T22:00:53.325 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2026-02-22T22:00:53.325 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:00:53.325 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T22:00:53.328 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:00:53.328 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_30 2026-02-22T22:00:53.328 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:2 2026-02-22T22:00:53.328 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:00:53.328 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T22:00:53.329 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:00:53.329 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:2 2026-02-22T22:00:53.329 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T22:00:53.329 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T22:00:53.329 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:00:53.329 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=2 2026-02-22T22:00:53.329 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:00:53.329 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T22:00:53.329 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:00:53.329 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=2 2026-02-22T22:00:53.329 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T22:00:53.329 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:00:53.330 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:2 2026-02-22T22:00:53.330 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T22:00:53.330 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T22:00:53.330 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:00:53.330 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:2 2026-02-22T22:00:53.330 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:00:53.330 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T22:00:53.330 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:00:53.330 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T22:00:53.330 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T22:00:53.330 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:00:53.330 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T22:00:53.331 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=2 2026-02-22T22:00:53.331 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=2 2026-02-22T22:00:53.331 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:00:53.339 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:00:53.339 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:00:53.339 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:00:53.339 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_30 2026-02-22T22:00:53.385 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2026-02-22T22:00:53.385 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:00:53.385 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T22:00:53.388 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:00:53.389 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_30 2026-02-22T22:00:53.389 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:3 2026-02-22T22:00:53.389 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:00:53.389 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T22:00:53.389 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T22:00:53.389 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:3 2026-02-22T22:00:53.389 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T22:00:53.389 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T22:00:53.389 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:00:53.389 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=3 2026-02-22T22:00:53.389 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T22:00:53.390 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T22:00:53.390 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:00:53.390 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=3 2026-02-22T22:00:53.390 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=3 2026-02-22T22:00:53.390 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T22:00:53.390 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:3 2026-02-22T22:00:53.390 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T22:00:53.390 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T22:00:53.390 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T22:00:53.390 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:3 2026-02-22T22:00:53.390 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:00:53.390 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T22:00:53.391 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:00:53.391 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=3 2026-02-22T22:00:53.391 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T22:00:53.391 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:00:53.391 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T22:00:53.391 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=3 2026-02-22T22:00:53.391 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=3 2026-02-22T22:00:53.391 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T22:00:53.399 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:00:53.399 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:00:53.399 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:00:53.399 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_30 2026-02-22T22:00:53.450 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ return 0 2026-02-22T22:00:53.451 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:00:53.451 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:00:53.451 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep -i Replaying 2026-02-22T22:00:53.452 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: "state": "Replaying" 2026-02-22T22:00:53.453 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_state=started 2026-02-22T22:00:53.453 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test started = started 2026-02-22T22:00:53.453 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:00:53.453 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_30 2026-02-22T22:00:53.454 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T22:00:53.454 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:00:53.454 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:00:53.454 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_30 2026-02-22T22:00:53.454 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = journal ']' 2026-02-22T22:00:53.454 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T22:00:53.454 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ mirror_image_snapshot cluster2 mirror image_30 2026-02-22T22:00:53.454 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:00:53.454 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:00:53.454 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_30 2026-02-22T22:00:53.454 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_30 2026-02-22T22:00:54.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Snapshot ID: 144 2026-02-22T22:00:54.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_30 2026-02-22T22:00:54.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T22:00:54.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:00:54.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:00:54.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_30 2026-02-22T22:00:54.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster2-mirror-image_30.status 2026-02-22T22:00:54.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster2-mirror-image_30.status 2026-02-22T22:00:54.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:00:54.213 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_log=/tmp/tmp.j44LZPowal/cluster2-mirror-image_30.status 2026-02-22T22:00:54.214 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster1-mirror-image_30.status 2026-02-22T22:00:54.214 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster1-mirror-image_30.status 2026-02-22T22:00:54.214 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:00:54.215 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_status_log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_30.status 2026-02-22T22:00:54.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_30 /tmp/tmp.j44LZPowal/cluster2-mirror-image_30.status 2026-02-22T22:00:54.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:00:54.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:00:54.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_30 2026-02-22T22:00:54.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster2-mirror-image_30.status 2026-02-22T22:00:54.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_30 --format xml 2026-02-22T22:00:54.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:00:54.243 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T22:00:54.245 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snapshot_id=144 2026-02-22T22:00:54.245 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T22:00:54.245 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:00:54.245 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.2 2026-02-22T22:00:54.447 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_30 /tmp/tmp.j44LZPowal/cluster1-mirror-image_30.status 2026-02-22T22:00:54.447 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:00:54.447 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:00:54.447 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_30 2026-02-22T22:00:54.447 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_30.status 2026-02-22T22:00:54.447 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_30 --format xml 2026-02-22T22:00:54.447 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:00:54.476 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T22:00:54.476 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T22:00:54.477 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T22:00:54.477 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T22:00:54.477 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T22:00:54.478 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T22:00:54.478 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 144 = '' 2026-02-22T22:00:54.478 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:00:54.478 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.4 2026-02-22T22:00:54.878 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_30 /tmp/tmp.j44LZPowal/cluster1-mirror-image_30.status 2026-02-22T22:00:54.878 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:00:54.878 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:00:54.879 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_30 2026-02-22T22:00:54.879 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_30.status 2026-02-22T22:00:54.879 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_30 --format xml 2026-02-22T22:00:54.879 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:00:54.908 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T22:00:54.908 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T22:00:54.909 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T22:00:54.909 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T22:00:54.909 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T22:00:54.910 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T22:00:54.910 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 144 = '' 2026-02-22T22:00:54.910 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:00:54.911 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.8 2026-02-22T22:00:55.711 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_30 /tmp/tmp.j44LZPowal/cluster1-mirror-image_30.status 2026-02-22T22:00:55.711 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:00:55.711 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:00:55.711 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_30 2026-02-22T22:00:55.711 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_30.status 2026-02-22T22:00:55.711 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_30 --format xml 2026-02-22T22:00:55.712 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:00:55.742 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T22:00:55.744 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id=144 2026-02-22T22:00:55.744 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 144 = 144 2026-02-22T22:00:55.744 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:00:55.744 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snap_present cluster1 mirror image_30 snap 2026-02-22T22:00:55.744 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:00:55.745 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:00:55.745 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_30 2026-02-22T22:00:55.745 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T22:00:55.745 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:00:55.745 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T22:00:55.745 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T22:00:56.744 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 info mirror/image_30@snap 2026-02-22T22:00:56.770 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_30': 2026-02-22T22:00:56.771 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 128 MiB in 32 objects 2026-02-22T22:00:56.771 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T22:00:56.771 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 2 2026-02-22T22:00:56.771 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 11bd740d0376 2026-02-22T22:00:56.771 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.11bd740d0376 2026-02-22T22:00:56.771 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T22:00:56.771 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock, non-primary 2026-02-22T22:00:56.771 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T22:00:56.771 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T22:00:56.772 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:52:56 2026 2026-02-22T22:00:56.772 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:52:56 2026 2026-02-22T22:00:56.772 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:52:56 2026 2026-02-22T22:00:56.772 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: protected: False 2026-02-22T22:00:56.772 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T22:00:56.772 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T22:00:56.772 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: cce0d5f1-d366-4b28-ba1a-4d3fe01fd2e1 2026-02-22T22:00:56.772 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: false 2026-02-22T22:00:56.773 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:00:56.774 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ compare_image_snaps mirror image_30 snap 2026-02-22T22:00:56.774 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:00:56.774 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_30 2026-02-22T22:00:56.774 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T22:00:56.774 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local ret=0 2026-02-22T22:00:56.774 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local rmt_export=/tmp/tmp.j44LZPowal/cluster2-mirror-image_30.export 2026-02-22T22:00:56.774 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local loc_export=/tmp/tmp.j44LZPowal/cluster1-mirror-image_30.export 2026-02-22T22:00:56.774 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-image_30.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_30.export 2026-02-22T22:00:56.774 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror export image_30@snap /tmp/tmp.j44LZPowal/cluster2-mirror-image_30.export 2026-02-22T22:00:56.880 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:00:56.887 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 -p mirror export image_30@snap /tmp/tmp.j44LZPowal/cluster1-mirror-image_30.export 2026-02-22T22:00:57.010 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:00:57.019 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ cmp /tmp/tmp.j44LZPowal/cluster2-mirror-image_30.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_30.export 2026-02-22T22:00:57.069 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-image_30.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_30.export 2026-02-22T22:00:57.104 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:00:57.104 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:00:57.104 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_31 2026-02-22T22:00:57.105 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_snap cluster2 mirror image_31 snap 2026-02-22T22:00:57.105 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:00:57.105 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:00:57.105 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_31 2026-02-22T22:00:57.105 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T22:00:57.105 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror snap create image_31@snap --debug-rbd=20 --debug-journaler=20 2026-02-22T22:00:58.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_started cluster1 mirror image_31 2026-02-22T22:00:58.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:00:58.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:00:58.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_31 2026-02-22T22:00:58.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_state cluster1 mirror image_31 started 2026-02-22T22:00:58.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:00:58.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:00:58.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_31 2026-02-22T22:00:58.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=started 2026-02-22T22:00:58.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:00:58.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:00:58.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T22:00:59.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_replay_state cluster1 mirror image_31 started 2026-02-22T22:00:59.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:00:59.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:00:59.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_31 2026-02-22T22:00:59.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=started 2026-02-22T22:00:59.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_result 2026-02-22T22:00:59.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=stopped 2026-02-22T22:00:59.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_31 2026-02-22T22:00:59.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_instance=cluster1 2026-02-22T22:00:59.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ grep -i state 2026-02-22T22:00:59.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:00:59.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:00:59.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance=cluster1 2026-02-22T22:00:59.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local loop_instance 2026-02-22T22:00:59.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:00:59.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sleep 0 2026-02-22T22:00:59.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:00:59.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ seq 0 3 2026-02-22T22:00:59.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:00:59.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_31 2026-02-22T22:00:59.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:0 2026-02-22T22:00:59.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:00:59.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T22:00:59.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:00:59.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:0 2026-02-22T22:00:59.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T22:00:59.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T22:00:59.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:00:59.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=0 2026-02-22T22:00:59.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:00:59.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T22:00:59.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:00:59.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=0 2026-02-22T22:00:59.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T22:00:59.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:00:59.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:00:59.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T22:00:59.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T22:00:59.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:00:59.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:0 2026-02-22T22:00:59.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:00:59.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T22:00:59.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:00:59.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T22:00:59.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:00:59.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:00:59.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T22:00:59.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=0 2026-02-22T22:00:59.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=0 2026-02-22T22:00:59.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:00:59.215 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:00:59.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:00:59.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:00:59.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_31 2026-02-22T22:00:59.261 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_31'] 2026-02-22T22:00:59.261 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:00:59.261 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T22:00:59.264 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:00:59.264 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_31 2026-02-22T22:00:59.264 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:1 2026-02-22T22:00:59.264 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:00:59.264 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T22:00:59.264 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:00:59.264 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:1 2026-02-22T22:00:59.264 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T22:00:59.264 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T22:00:59.265 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:00:59.265 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=1 2026-02-22T22:00:59.265 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:00:59.265 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T22:00:59.265 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:00:59.265 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=1 2026-02-22T22:00:59.265 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T22:00:59.265 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:00:59.265 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:1 2026-02-22T22:00:59.265 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T22:00:59.265 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T22:00:59.265 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:00:59.266 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:1 2026-02-22T22:00:59.266 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:00:59.266 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T22:00:59.266 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:00:59.266 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T22:00:59.266 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T22:00:59.266 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:00:59.266 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T22:00:59.266 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=1 2026-02-22T22:00:59.266 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=1 2026-02-22T22:00:59.266 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:00:59.275 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:00:59.275 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:00:59.275 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:00:59.275 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_31 2026-02-22T22:00:59.319 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_31'] 2026-02-22T22:00:59.320 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:00:59.320 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T22:00:59.322 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:00:59.322 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_31 2026-02-22T22:00:59.323 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:2 2026-02-22T22:00:59.323 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:00:59.323 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T22:00:59.323 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:00:59.323 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:2 2026-02-22T22:00:59.323 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T22:00:59.323 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T22:00:59.323 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:00:59.323 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=2 2026-02-22T22:00:59.323 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:00:59.323 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T22:00:59.323 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:00:59.324 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=2 2026-02-22T22:00:59.324 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T22:00:59.324 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:00:59.324 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:2 2026-02-22T22:00:59.324 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T22:00:59.324 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T22:00:59.324 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:00:59.324 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:2 2026-02-22T22:00:59.324 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:00:59.324 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T22:00:59.324 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:00:59.324 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T22:00:59.325 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T22:00:59.325 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:00:59.325 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T22:00:59.325 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=2 2026-02-22T22:00:59.325 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=2 2026-02-22T22:00:59.325 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:00:59.333 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:00:59.334 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:00:59.334 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:00:59.334 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_31 2026-02-22T22:00:59.383 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ return 0 2026-02-22T22:00:59.384 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:00:59.384 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:00:59.384 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep -i Replaying 2026-02-22T22:00:59.385 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: "state": "Replaying" 2026-02-22T22:00:59.386 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_state=started 2026-02-22T22:00:59.386 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test started = started 2026-02-22T22:00:59.386 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:00:59.386 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_31 2026-02-22T22:00:59.386 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T22:00:59.386 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:00:59.386 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:00:59.387 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_31 2026-02-22T22:00:59.387 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = journal ']' 2026-02-22T22:00:59.387 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T22:00:59.387 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ mirror_image_snapshot cluster2 mirror image_31 2026-02-22T22:00:59.387 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:00:59.387 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:00:59.387 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_31 2026-02-22T22:00:59.387 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_31 2026-02-22T22:01:00.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Snapshot ID: 146 2026-02-22T22:01:00.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_31 2026-02-22T22:01:00.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T22:01:00.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:01:00.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:00.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_31 2026-02-22T22:01:00.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster2-mirror-image_31.status 2026-02-22T22:01:00.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster2-mirror-image_31.status 2026-02-22T22:01:00.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:01:00.209 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_log=/tmp/tmp.j44LZPowal/cluster2-mirror-image_31.status 2026-02-22T22:01:00.209 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster1-mirror-image_31.status 2026-02-22T22:01:00.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster1-mirror-image_31.status 2026-02-22T22:01:00.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:01:00.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_status_log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_31.status 2026-02-22T22:01:00.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_31 /tmp/tmp.j44LZPowal/cluster2-mirror-image_31.status 2026-02-22T22:01:00.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:01:00.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:00.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_31 2026-02-22T22:01:00.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster2-mirror-image_31.status 2026-02-22T22:01:00.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_31 --format xml 2026-02-22T22:01:00.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:01:00.239 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T22:01:00.240 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snapshot_id=146 2026-02-22T22:01:00.240 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T22:01:00.240 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:01:00.241 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.2 2026-02-22T22:01:00.442 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_31 /tmp/tmp.j44LZPowal/cluster1-mirror-image_31.status 2026-02-22T22:01:00.442 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:01:00.442 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:00.442 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_31 2026-02-22T22:01:00.443 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_31.status 2026-02-22T22:01:00.443 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_31 --format xml 2026-02-22T22:01:00.443 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:01:00.472 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T22:01:00.472 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T22:01:00.473 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T22:01:00.473 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T22:01:00.473 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T22:01:00.474 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T22:01:00.474 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 146 = '' 2026-02-22T22:01:00.474 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:01:00.474 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.4 2026-02-22T22:01:00.875 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_31 /tmp/tmp.j44LZPowal/cluster1-mirror-image_31.status 2026-02-22T22:01:00.876 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:01:00.876 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:00.876 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_31 2026-02-22T22:01:00.876 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_31.status 2026-02-22T22:01:00.876 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_31 --format xml 2026-02-22T22:01:00.876 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:01:00.905 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T22:01:00.906 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T22:01:00.907 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T22:01:00.907 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T22:01:00.907 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T22:01:00.907 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T22:01:00.908 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 146 = '' 2026-02-22T22:01:00.908 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:01:00.908 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.8 2026-02-22T22:01:01.709 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_31 /tmp/tmp.j44LZPowal/cluster1-mirror-image_31.status 2026-02-22T22:01:01.709 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:01:01.709 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:01.709 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_31 2026-02-22T22:01:01.709 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_31.status 2026-02-22T22:01:01.709 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_31 --format xml 2026-02-22T22:01:01.709 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:01:01.741 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T22:01:01.743 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id=146 2026-02-22T22:01:01.743 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 146 = 146 2026-02-22T22:01:01.743 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:01:01.743 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snap_present cluster1 mirror image_31 snap 2026-02-22T22:01:01.743 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:01:01.743 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:01.743 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_31 2026-02-22T22:01:01.744 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T22:01:01.744 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:01:01.744 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T22:01:01.744 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T22:01:02.743 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 info mirror/image_31@snap 2026-02-22T22:01:02.769 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_31': 2026-02-22T22:01:02.770 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 128 MiB in 32 objects 2026-02-22T22:01:02.770 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T22:01:02.770 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 2 2026-02-22T22:01:02.770 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 122750858afc 2026-02-22T22:01:02.770 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.122750858afc 2026-02-22T22:01:02.770 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T22:01:02.770 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock, non-primary 2026-02-22T22:01:02.770 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T22:01:02.770 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T22:01:02.770 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:53:04 2026 2026-02-22T22:01:02.771 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:53:04 2026 2026-02-22T22:01:02.771 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:53:04 2026 2026-02-22T22:01:02.771 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: protected: False 2026-02-22T22:01:02.771 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T22:01:02.771 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T22:01:02.771 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: 97a4414e-20e7-4288-9c80-0dce42f41d4f 2026-02-22T22:01:02.771 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: false 2026-02-22T22:01:02.772 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:01:02.772 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ compare_image_snaps mirror image_31 snap 2026-02-22T22:01:02.773 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:02.773 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_31 2026-02-22T22:01:02.773 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T22:01:02.773 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local ret=0 2026-02-22T22:01:02.773 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local rmt_export=/tmp/tmp.j44LZPowal/cluster2-mirror-image_31.export 2026-02-22T22:01:02.773 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local loc_export=/tmp/tmp.j44LZPowal/cluster1-mirror-image_31.export 2026-02-22T22:01:02.773 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-image_31.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_31.export 2026-02-22T22:01:02.773 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror export image_31@snap /tmp/tmp.j44LZPowal/cluster2-mirror-image_31.export 2026-02-22T22:01:02.875 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:01:02.883 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 -p mirror export image_31@snap /tmp/tmp.j44LZPowal/cluster1-mirror-image_31.export 2026-02-22T22:01:03.013 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:01:03.021 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ cmp /tmp/tmp.j44LZPowal/cluster2-mirror-image_31.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_31.export 2026-02-22T22:01:03.071 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-image_31.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_31.export 2026-02-22T22:01:03.105 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:01:03.106 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:01:03.106 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_32 2026-02-22T22:01:03.106 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_snap cluster2 mirror image_32 snap 2026-02-22T22:01:03.106 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:01:03.106 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:03.106 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_32 2026-02-22T22:01:03.106 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T22:01:03.106 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror snap create image_32@snap --debug-rbd=20 --debug-journaler=20 2026-02-22T22:01:04.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_started cluster1 mirror image_32 2026-02-22T22:01:04.209 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:01:04.209 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:04.209 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_32 2026-02-22T22:01:04.209 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_state cluster1 mirror image_32 started 2026-02-22T22:01:04.209 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:01:04.209 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:04.209 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_32 2026-02-22T22:01:04.209 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=started 2026-02-22T22:01:04.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:01:04.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:01:04.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T22:01:05.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_replay_state cluster1 mirror image_32 started 2026-02-22T22:01:05.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:01:05.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:05.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_32 2026-02-22T22:01:05.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=started 2026-02-22T22:01:05.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_result 2026-02-22T22:01:05.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=stopped 2026-02-22T22:01:05.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_32 2026-02-22T22:01:05.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_instance=cluster1 2026-02-22T22:01:05.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ grep -i state 2026-02-22T22:01:05.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:01:05.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:01:05.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance=cluster1 2026-02-22T22:01:05.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local loop_instance 2026-02-22T22:01:05.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:01:05.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sleep 0 2026-02-22T22:01:05.214 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:01:05.215 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ seq 0 3 2026-02-22T22:01:05.215 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:01:05.215 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_32 2026-02-22T22:01:05.215 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:0 2026-02-22T22:01:05.215 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:01:05.215 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T22:01:05.215 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:01:05.215 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:0 2026-02-22T22:01:05.215 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T22:01:05.215 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T22:01:05.215 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:01:05.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=0 2026-02-22T22:01:05.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:01:05.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T22:01:05.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:01:05.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=0 2026-02-22T22:01:05.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T22:01:05.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:01:05.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:01:05.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T22:01:05.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T22:01:05.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:01:05.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:0 2026-02-22T22:01:05.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:01:05.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T22:01:05.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:01:05.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T22:01:05.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:01:05.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:01:05.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T22:01:05.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=0 2026-02-22T22:01:05.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=0 2026-02-22T22:01:05.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:01:05.226 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:01:05.226 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:01:05.226 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:01:05.226 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_32 2026-02-22T22:01:05.273 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_32'] 2026-02-22T22:01:05.274 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:01:05.274 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T22:01:05.277 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:01:05.277 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_32 2026-02-22T22:01:05.277 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:1 2026-02-22T22:01:05.277 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:01:05.277 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T22:01:05.277 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:01:05.277 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:1 2026-02-22T22:01:05.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T22:01:05.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T22:01:05.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:01:05.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=1 2026-02-22T22:01:05.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:01:05.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T22:01:05.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:01:05.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=1 2026-02-22T22:01:05.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T22:01:05.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:01:05.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:1 2026-02-22T22:01:05.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T22:01:05.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T22:01:05.279 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:01:05.279 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:1 2026-02-22T22:01:05.279 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:01:05.279 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T22:01:05.279 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:01:05.279 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T22:01:05.279 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T22:01:05.279 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:01:05.279 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T22:01:05.279 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=1 2026-02-22T22:01:05.279 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=1 2026-02-22T22:01:05.279 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:01:05.287 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:01:05.287 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:01:05.287 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:01:05.287 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_32 2026-02-22T22:01:05.334 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_32'] 2026-02-22T22:01:05.334 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:01:05.334 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T22:01:05.338 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:01:05.338 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_32 2026-02-22T22:01:05.338 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:2 2026-02-22T22:01:05.338 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:01:05.338 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T22:01:05.338 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:01:05.338 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:2 2026-02-22T22:01:05.338 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T22:01:05.339 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T22:01:05.339 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:01:05.339 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=2 2026-02-22T22:01:05.339 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:01:05.339 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T22:01:05.339 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:01:05.339 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=2 2026-02-22T22:01:05.339 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T22:01:05.339 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:01:05.339 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:2 2026-02-22T22:01:05.339 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T22:01:05.339 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T22:01:05.339 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:01:05.340 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:2 2026-02-22T22:01:05.340 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:01:05.340 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T22:01:05.340 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:01:05.340 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T22:01:05.340 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T22:01:05.340 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:01:05.340 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T22:01:05.340 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=2 2026-02-22T22:01:05.340 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=2 2026-02-22T22:01:05.340 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:01:05.348 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:01:05.349 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:01:05.349 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:01:05.349 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_32 2026-02-22T22:01:05.401 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ return 0 2026-02-22T22:01:05.402 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:01:05.402 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:01:05.402 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep -i Replaying 2026-02-22T22:01:05.403 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: "state": "Replaying" 2026-02-22T22:01:05.404 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_state=started 2026-02-22T22:01:05.404 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test started = started 2026-02-22T22:01:05.404 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:01:05.405 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_32 2026-02-22T22:01:05.405 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T22:01:05.405 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:01:05.405 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:05.405 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_32 2026-02-22T22:01:05.405 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = journal ']' 2026-02-22T22:01:05.405 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T22:01:05.405 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ mirror_image_snapshot cluster2 mirror image_32 2026-02-22T22:01:05.405 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:01:05.405 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:05.405 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_32 2026-02-22T22:01:05.406 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_32 2026-02-22T22:01:06.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Snapshot ID: 148 2026-02-22T22:01:06.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_32 2026-02-22T22:01:06.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T22:01:06.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:01:06.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:06.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_32 2026-02-22T22:01:06.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster2-mirror-image_32.status 2026-02-22T22:01:06.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster2-mirror-image_32.status 2026-02-22T22:01:06.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:01:06.218 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_log=/tmp/tmp.j44LZPowal/cluster2-mirror-image_32.status 2026-02-22T22:01:06.219 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster1-mirror-image_32.status 2026-02-22T22:01:06.219 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster1-mirror-image_32.status 2026-02-22T22:01:06.219 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:01:06.220 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_status_log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_32.status 2026-02-22T22:01:06.220 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_32 /tmp/tmp.j44LZPowal/cluster2-mirror-image_32.status 2026-02-22T22:01:06.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:01:06.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:06.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_32 2026-02-22T22:01:06.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster2-mirror-image_32.status 2026-02-22T22:01:06.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_32 --format xml 2026-02-22T22:01:06.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:01:06.248 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T22:01:06.250 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snapshot_id=148 2026-02-22T22:01:06.250 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T22:01:06.250 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:01:06.251 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.2 2026-02-22T22:01:06.451 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_32 /tmp/tmp.j44LZPowal/cluster1-mirror-image_32.status 2026-02-22T22:01:06.452 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:01:06.452 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:06.452 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_32 2026-02-22T22:01:06.452 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_32.status 2026-02-22T22:01:06.452 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_32 --format xml 2026-02-22T22:01:06.452 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:01:06.481 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T22:01:06.481 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T22:01:06.482 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T22:01:06.482 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T22:01:06.482 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T22:01:06.483 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T22:01:06.483 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 148 = '' 2026-02-22T22:01:06.483 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:01:06.483 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.4 2026-02-22T22:01:06.884 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_32 /tmp/tmp.j44LZPowal/cluster1-mirror-image_32.status 2026-02-22T22:01:06.884 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:01:06.884 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:06.884 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_32 2026-02-22T22:01:06.884 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_32.status 2026-02-22T22:01:06.885 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_32 --format xml 2026-02-22T22:01:06.885 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:01:06.913 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T22:01:06.913 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T22:01:06.914 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T22:01:06.914 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T22:01:06.914 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T22:01:06.915 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T22:01:06.915 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 148 = '' 2026-02-22T22:01:06.915 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:01:06.915 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.8 2026-02-22T22:01:07.716 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_32 /tmp/tmp.j44LZPowal/cluster1-mirror-image_32.status 2026-02-22T22:01:07.716 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:01:07.716 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:07.717 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_32 2026-02-22T22:01:07.717 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_32.status 2026-02-22T22:01:07.717 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_32 --format xml 2026-02-22T22:01:07.717 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:01:07.749 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T22:01:07.751 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id=148 2026-02-22T22:01:07.751 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 148 = 148 2026-02-22T22:01:07.751 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:01:07.751 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snap_present cluster1 mirror image_32 snap 2026-02-22T22:01:07.751 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:01:07.751 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:07.751 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_32 2026-02-22T22:01:07.751 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T22:01:07.751 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:01:07.752 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T22:01:07.752 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T22:01:08.752 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 info mirror/image_32@snap 2026-02-22T22:01:08.782 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_32': 2026-02-22T22:01:08.782 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 128 MiB in 32 objects 2026-02-22T22:01:08.782 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T22:01:08.782 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 2 2026-02-22T22:01:08.782 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 1248450e7b09 2026-02-22T22:01:08.782 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.1248450e7b09 2026-02-22T22:01:08.782 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T22:01:08.783 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock, non-primary 2026-02-22T22:01:08.783 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T22:01:08.783 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T22:01:08.783 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:53:11 2026 2026-02-22T22:01:08.783 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:53:11 2026 2026-02-22T22:01:08.783 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:53:11 2026 2026-02-22T22:01:08.783 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: protected: False 2026-02-22T22:01:08.783 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T22:01:08.783 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T22:01:08.783 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: 533da63c-6a64-4a63-a224-d042b1c3553d 2026-02-22T22:01:08.783 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: false 2026-02-22T22:01:08.785 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:01:08.785 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ compare_image_snaps mirror image_32 snap 2026-02-22T22:01:08.785 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:08.785 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_32 2026-02-22T22:01:08.785 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T22:01:08.785 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local ret=0 2026-02-22T22:01:08.785 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local rmt_export=/tmp/tmp.j44LZPowal/cluster2-mirror-image_32.export 2026-02-22T22:01:08.786 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local loc_export=/tmp/tmp.j44LZPowal/cluster1-mirror-image_32.export 2026-02-22T22:01:08.786 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-image_32.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_32.export 2026-02-22T22:01:08.786 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror export image_32@snap /tmp/tmp.j44LZPowal/cluster2-mirror-image_32.export 2026-02-22T22:01:08.892 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:01:08.901 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 -p mirror export image_32@snap /tmp/tmp.j44LZPowal/cluster1-mirror-image_32.export 2026-02-22T22:01:09.042 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:01:09.051 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ cmp /tmp/tmp.j44LZPowal/cluster2-mirror-image_32.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_32.export 2026-02-22T22:01:09.100 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-image_32.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_32.export 2026-02-22T22:01:09.135 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:01:09.135 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:01:09.136 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_33 2026-02-22T22:01:09.136 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_snap cluster2 mirror image_33 snap 2026-02-22T22:01:09.136 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:01:09.136 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:09.136 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_33 2026-02-22T22:01:09.136 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T22:01:09.136 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror snap create image_33@snap --debug-rbd=20 --debug-journaler=20 2026-02-22T22:01:10.231 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_started cluster1 mirror image_33 2026-02-22T22:01:10.231 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:01:10.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:10.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_33 2026-02-22T22:01:10.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_state cluster1 mirror image_33 started 2026-02-22T22:01:10.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:01:10.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:10.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_33 2026-02-22T22:01:10.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=started 2026-02-22T22:01:10.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:01:10.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:01:10.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T22:01:11.233 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_replay_state cluster1 mirror image_33 started 2026-02-22T22:01:11.233 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:01:11.233 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:11.233 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_33 2026-02-22T22:01:11.233 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=started 2026-02-22T22:01:11.233 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_result 2026-02-22T22:01:11.234 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=stopped 2026-02-22T22:01:11.234 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_33 2026-02-22T22:01:11.234 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_instance=cluster1 2026-02-22T22:01:11.234 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:01:11.234 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:01:11.234 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance=cluster1 2026-02-22T22:01:11.234 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local loop_instance 2026-02-22T22:01:11.234 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ grep -i state 2026-02-22T22:01:11.234 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:01:11.234 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sleep 0 2026-02-22T22:01:11.237 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:01:11.237 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ seq 0 3 2026-02-22T22:01:11.237 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:01:11.237 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_33 2026-02-22T22:01:11.237 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:0 2026-02-22T22:01:11.238 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:01:11.238 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T22:01:11.238 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:01:11.238 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:0 2026-02-22T22:01:11.238 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T22:01:11.238 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T22:01:11.238 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:01:11.238 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=0 2026-02-22T22:01:11.238 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:01:11.238 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T22:01:11.238 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:01:11.238 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=0 2026-02-22T22:01:11.239 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T22:01:11.239 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:01:11.239 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:01:11.239 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T22:01:11.239 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T22:01:11.239 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:01:11.239 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:0 2026-02-22T22:01:11.239 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:01:11.239 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T22:01:11.239 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:01:11.239 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T22:01:11.239 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:01:11.240 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:01:11.240 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T22:01:11.240 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=0 2026-02-22T22:01:11.240 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=0 2026-02-22T22:01:11.240 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:01:11.248 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:01:11.248 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:01:11.248 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:01:11.248 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_33 2026-02-22T22:01:11.298 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ return 0 2026-02-22T22:01:11.299 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:01:11.299 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:01:11.299 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep -i Replaying 2026-02-22T22:01:11.300 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: "state": "Replaying" 2026-02-22T22:01:11.301 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_state=started 2026-02-22T22:01:11.301 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test started = started 2026-02-22T22:01:11.301 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:01:11.301 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_33 2026-02-22T22:01:11.301 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T22:01:11.301 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:01:11.301 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:11.302 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_33 2026-02-22T22:01:11.302 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = journal ']' 2026-02-22T22:01:11.302 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T22:01:11.302 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ mirror_image_snapshot cluster2 mirror image_33 2026-02-22T22:01:11.302 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:01:11.302 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:11.302 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_33 2026-02-22T22:01:11.302 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_33 2026-02-22T22:01:12.233 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Snapshot ID: 150 2026-02-22T22:01:12.239 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_33 2026-02-22T22:01:12.239 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T22:01:12.240 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:01:12.240 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:12.240 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_33 2026-02-22T22:01:12.240 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster2-mirror-image_33.status 2026-02-22T22:01:12.240 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster2-mirror-image_33.status 2026-02-22T22:01:12.240 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:01:12.241 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_log=/tmp/tmp.j44LZPowal/cluster2-mirror-image_33.status 2026-02-22T22:01:12.241 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster1-mirror-image_33.status 2026-02-22T22:01:12.242 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster1-mirror-image_33.status 2026-02-22T22:01:12.242 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:01:12.244 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_status_log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_33.status 2026-02-22T22:01:12.244 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_33 /tmp/tmp.j44LZPowal/cluster2-mirror-image_33.status 2026-02-22T22:01:12.244 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:01:12.244 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:12.244 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_33 2026-02-22T22:01:12.244 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster2-mirror-image_33.status 2026-02-22T22:01:12.244 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_33 --format xml 2026-02-22T22:01:12.245 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:01:12.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T22:01:12.273 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snapshot_id=150 2026-02-22T22:01:12.273 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T22:01:12.273 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:01:12.273 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.2 2026-02-22T22:01:12.475 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_33 /tmp/tmp.j44LZPowal/cluster1-mirror-image_33.status 2026-02-22T22:01:12.475 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:01:12.475 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:12.475 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_33 2026-02-22T22:01:12.476 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_33.status 2026-02-22T22:01:12.476 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_33 --format xml 2026-02-22T22:01:12.476 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:01:12.504 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T22:01:12.504 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T22:01:12.505 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T22:01:12.505 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T22:01:12.505 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T22:01:12.506 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T22:01:12.506 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 150 = '' 2026-02-22T22:01:12.506 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:01:12.506 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.4 2026-02-22T22:01:12.907 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_33 /tmp/tmp.j44LZPowal/cluster1-mirror-image_33.status 2026-02-22T22:01:12.908 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:01:12.908 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:12.908 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_33 2026-02-22T22:01:12.908 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_33.status 2026-02-22T22:01:12.908 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_33 --format xml 2026-02-22T22:01:12.908 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:01:12.938 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T22:01:12.938 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T22:01:12.939 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T22:01:12.939 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T22:01:12.939 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T22:01:12.940 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T22:01:12.940 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 150 = '' 2026-02-22T22:01:12.941 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:01:12.941 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.8 2026-02-22T22:01:13.741 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_33 /tmp/tmp.j44LZPowal/cluster1-mirror-image_33.status 2026-02-22T22:01:13.741 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:01:13.741 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:13.741 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_33 2026-02-22T22:01:13.741 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_33.status 2026-02-22T22:01:13.741 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_33 --format xml 2026-02-22T22:01:13.741 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:01:13.771 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T22:01:13.773 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id=150 2026-02-22T22:01:13.774 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 150 = 150 2026-02-22T22:01:13.774 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:01:13.774 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snap_present cluster1 mirror image_33 snap 2026-02-22T22:01:13.774 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:01:13.774 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:13.774 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_33 2026-02-22T22:01:13.774 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T22:01:13.774 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:01:13.774 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T22:01:13.774 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T22:01:14.774 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 info mirror/image_33@snap 2026-02-22T22:01:14.801 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_33': 2026-02-22T22:01:14.802 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 128 MiB in 32 objects 2026-02-22T22:01:14.802 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T22:01:14.802 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 2 2026-02-22T22:01:14.802 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 119a7f47f547 2026-02-22T22:01:14.802 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.119a7f47f547 2026-02-22T22:01:14.802 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T22:01:14.802 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock, non-primary 2026-02-22T22:01:14.802 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T22:01:14.802 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T22:01:14.802 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:53:21 2026 2026-02-22T22:01:14.802 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:53:21 2026 2026-02-22T22:01:14.803 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:53:21 2026 2026-02-22T22:01:14.803 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: protected: False 2026-02-22T22:01:14.803 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T22:01:14.803 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T22:01:14.803 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: ad9fe32c-2e40-4486-ab81-178c7a9c3a27 2026-02-22T22:01:14.803 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: false 2026-02-22T22:01:14.804 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:01:14.804 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ compare_image_snaps mirror image_33 snap 2026-02-22T22:01:14.804 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:14.805 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_33 2026-02-22T22:01:14.805 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T22:01:14.805 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local ret=0 2026-02-22T22:01:14.805 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local rmt_export=/tmp/tmp.j44LZPowal/cluster2-mirror-image_33.export 2026-02-22T22:01:14.805 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local loc_export=/tmp/tmp.j44LZPowal/cluster1-mirror-image_33.export 2026-02-22T22:01:14.805 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-image_33.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_33.export 2026-02-22T22:01:14.805 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror export image_33@snap /tmp/tmp.j44LZPowal/cluster2-mirror-image_33.export 2026-02-22T22:01:14.909 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:01:14.916 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 -p mirror export image_33@snap /tmp/tmp.j44LZPowal/cluster1-mirror-image_33.export 2026-02-22T22:01:15.039 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:01:15.045 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ cmp /tmp/tmp.j44LZPowal/cluster2-mirror-image_33.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_33.export 2026-02-22T22:01:15.093 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-image_33.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_33.export 2026-02-22T22:01:15.127 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:01:15.127 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:01:15.127 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_34 2026-02-22T22:01:15.127 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_snap cluster2 mirror image_34 snap 2026-02-22T22:01:15.127 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:01:15.127 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:15.128 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_34 2026-02-22T22:01:15.128 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T22:01:15.128 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror snap create image_34@snap --debug-rbd=20 --debug-journaler=20 2026-02-22T22:01:16.865 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_started cluster1 mirror image_34 2026-02-22T22:01:16.865 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:01:16.865 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:16.865 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_34 2026-02-22T22:01:16.865 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_state cluster1 mirror image_34 started 2026-02-22T22:01:16.865 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:01:16.865 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:16.865 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_34 2026-02-22T22:01:16.866 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=started 2026-02-22T22:01:16.866 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:01:16.866 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:01:16.866 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T22:01:17.865 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_replay_state cluster1 mirror image_34 started 2026-02-22T22:01:17.865 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:01:17.865 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:17.865 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_34 2026-02-22T22:01:17.865 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=started 2026-02-22T22:01:17.865 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_result 2026-02-22T22:01:17.865 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=stopped 2026-02-22T22:01:17.866 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_34 2026-02-22T22:01:17.866 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_instance=cluster1 2026-02-22T22:01:17.866 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ grep -i state 2026-02-22T22:01:17.866 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:01:17.866 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:01:17.866 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance=cluster1 2026-02-22T22:01:17.866 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local loop_instance 2026-02-22T22:01:17.866 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:01:17.866 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sleep 0 2026-02-22T22:01:17.867 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:01:17.868 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ seq 0 3 2026-02-22T22:01:17.869 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:01:17.869 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_34 2026-02-22T22:01:17.869 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:0 2026-02-22T22:01:17.869 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:01:17.869 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T22:01:17.869 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:01:17.869 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:0 2026-02-22T22:01:17.869 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T22:01:17.869 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T22:01:17.869 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:01:17.869 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=0 2026-02-22T22:01:17.869 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:01:17.869 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T22:01:17.869 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:01:17.869 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=0 2026-02-22T22:01:17.869 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T22:01:17.870 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:01:17.870 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:01:17.870 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T22:01:17.870 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T22:01:17.870 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:01:17.870 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:0 2026-02-22T22:01:17.870 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:01:17.870 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T22:01:17.870 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:01:17.870 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T22:01:17.870 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:01:17.870 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:01:17.870 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T22:01:17.870 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=0 2026-02-22T22:01:17.870 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=0 2026-02-22T22:01:17.870 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:01:17.883 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:01:17.883 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:01:17.884 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:01:17.884 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_34 2026-02-22T22:01:17.930 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_34'] 2026-02-22T22:01:17.930 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:01:17.930 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T22:01:17.933 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:01:17.933 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_34 2026-02-22T22:01:17.933 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:1 2026-02-22T22:01:17.934 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:01:17.934 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T22:01:17.934 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:01:17.934 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:1 2026-02-22T22:01:17.934 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T22:01:17.934 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T22:01:17.934 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:01:17.934 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=1 2026-02-22T22:01:17.934 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:01:17.934 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T22:01:17.934 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:01:17.935 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=1 2026-02-22T22:01:17.935 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T22:01:17.935 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:01:17.935 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:1 2026-02-22T22:01:17.935 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T22:01:17.935 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T22:01:17.935 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:01:17.935 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:1 2026-02-22T22:01:17.935 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:01:17.935 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T22:01:17.935 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:01:17.935 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T22:01:17.935 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T22:01:17.936 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:01:17.936 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T22:01:17.936 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=1 2026-02-22T22:01:17.936 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=1 2026-02-22T22:01:17.936 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:01:17.945 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:01:17.946 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:01:17.946 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:01:17.946 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_34 2026-02-22T22:01:17.993 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_34'] 2026-02-22T22:01:17.993 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:01:17.993 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T22:01:17.995 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:01:17.995 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_34 2026-02-22T22:01:17.995 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:2 2026-02-22T22:01:17.995 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:01:17.995 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T22:01:17.995 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:01:17.996 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:2 2026-02-22T22:01:17.996 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T22:01:17.996 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T22:01:17.996 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:01:17.996 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=2 2026-02-22T22:01:17.996 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:01:17.996 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T22:01:17.996 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:01:17.996 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=2 2026-02-22T22:01:17.996 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T22:01:17.996 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:01:17.997 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:2 2026-02-22T22:01:17.997 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T22:01:17.997 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T22:01:17.997 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:01:17.997 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:2 2026-02-22T22:01:17.997 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:01:17.997 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T22:01:17.997 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:01:17.997 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T22:01:17.997 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T22:01:17.997 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:01:17.997 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T22:01:17.997 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=2 2026-02-22T22:01:17.997 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=2 2026-02-22T22:01:17.997 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:01:18.009 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:01:18.009 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:01:18.009 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:01:18.009 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_34 2026-02-22T22:01:18.054 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_34'] 2026-02-22T22:01:18.054 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:01:18.054 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T22:01:18.057 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:01:18.057 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_34 2026-02-22T22:01:18.057 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:3 2026-02-22T22:01:18.058 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:01:18.058 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T22:01:18.058 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T22:01:18.058 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:3 2026-02-22T22:01:18.058 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T22:01:18.058 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T22:01:18.058 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:01:18.058 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=3 2026-02-22T22:01:18.058 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T22:01:18.058 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T22:01:18.058 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:01:18.059 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=3 2026-02-22T22:01:18.059 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=3 2026-02-22T22:01:18.059 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T22:01:18.059 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:3 2026-02-22T22:01:18.059 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T22:01:18.059 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T22:01:18.059 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T22:01:18.059 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:3 2026-02-22T22:01:18.059 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:01:18.059 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T22:01:18.059 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:01:18.059 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=3 2026-02-22T22:01:18.060 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T22:01:18.060 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:01:18.060 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T22:01:18.060 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=3 2026-02-22T22:01:18.060 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=3 2026-02-22T22:01:18.060 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T22:01:18.068 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:01:18.069 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:01:18.069 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:01:18.069 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_34 2026-02-22T22:01:18.119 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ return 0 2026-02-22T22:01:18.120 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:01:18.120 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:01:18.120 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep -i Replaying 2026-02-22T22:01:18.121 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: "state": "Replaying" 2026-02-22T22:01:18.122 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_state=started 2026-02-22T22:01:18.123 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test started = started 2026-02-22T22:01:18.123 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:01:18.123 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_34 2026-02-22T22:01:18.123 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T22:01:18.123 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:01:18.123 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:18.123 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_34 2026-02-22T22:01:18.123 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = journal ']' 2026-02-22T22:01:18.123 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T22:01:18.123 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ mirror_image_snapshot cluster2 mirror image_34 2026-02-22T22:01:18.124 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:01:18.124 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:18.124 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_34 2026-02-22T22:01:18.124 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_34 2026-02-22T22:01:18.870 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Snapshot ID: 152 2026-02-22T22:01:18.875 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_34 2026-02-22T22:01:18.875 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T22:01:18.875 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:01:18.875 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:18.875 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_34 2026-02-22T22:01:18.875 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster2-mirror-image_34.status 2026-02-22T22:01:18.876 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster2-mirror-image_34.status 2026-02-22T22:01:18.876 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:01:18.876 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_log=/tmp/tmp.j44LZPowal/cluster2-mirror-image_34.status 2026-02-22T22:01:18.877 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster1-mirror-image_34.status 2026-02-22T22:01:18.877 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster1-mirror-image_34.status 2026-02-22T22:01:18.877 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:01:18.879 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_status_log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_34.status 2026-02-22T22:01:18.879 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_34 /tmp/tmp.j44LZPowal/cluster2-mirror-image_34.status 2026-02-22T22:01:18.879 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:01:18.879 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:18.879 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_34 2026-02-22T22:01:18.879 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster2-mirror-image_34.status 2026-02-22T22:01:18.879 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_34 --format xml 2026-02-22T22:01:18.879 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:01:18.906 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T22:01:18.907 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snapshot_id=152 2026-02-22T22:01:18.908 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T22:01:18.908 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:01:18.908 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.2 2026-02-22T22:01:19.109 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_34 /tmp/tmp.j44LZPowal/cluster1-mirror-image_34.status 2026-02-22T22:01:19.109 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:01:19.109 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:19.109 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_34 2026-02-22T22:01:19.109 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_34.status 2026-02-22T22:01:19.109 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_34 --format xml 2026-02-22T22:01:19.110 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:01:19.139 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T22:01:19.140 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id=68 2026-02-22T22:01:19.140 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 152 = 68 2026-02-22T22:01:19.140 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:01:19.140 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.4 2026-02-22T22:01:19.541 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_34 /tmp/tmp.j44LZPowal/cluster1-mirror-image_34.status 2026-02-22T22:01:19.542 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:01:19.542 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:19.542 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_34 2026-02-22T22:01:19.542 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_34.status 2026-02-22T22:01:19.542 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_34 --format xml 2026-02-22T22:01:19.542 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:01:19.574 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T22:01:19.574 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T22:01:19.575 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T22:01:19.575 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T22:01:19.575 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T22:01:19.576 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T22:01:19.576 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 152 = '' 2026-02-22T22:01:19.576 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:01:19.576 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.8 2026-02-22T22:01:20.377 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_34 /tmp/tmp.j44LZPowal/cluster1-mirror-image_34.status 2026-02-22T22:01:20.377 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:01:20.377 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:20.377 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_34 2026-02-22T22:01:20.377 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_34.status 2026-02-22T22:01:20.377 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_34 --format xml 2026-02-22T22:01:20.377 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:01:20.405 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T22:01:20.406 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T22:01:20.407 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T22:01:20.407 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T22:01:20.407 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T22:01:20.408 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T22:01:20.408 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 152 = '' 2026-02-22T22:01:20.408 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:01:20.408 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1.6 2026-02-22T22:01:22.009 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_34 /tmp/tmp.j44LZPowal/cluster1-mirror-image_34.status 2026-02-22T22:01:22.009 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:01:22.009 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:22.009 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_34 2026-02-22T22:01:22.009 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_34.status 2026-02-22T22:01:22.010 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_34 --format xml 2026-02-22T22:01:22.010 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:01:22.039 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T22:01:22.041 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id=152 2026-02-22T22:01:22.042 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 152 = 152 2026-02-22T22:01:22.042 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:01:22.042 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snap_present cluster1 mirror image_34 snap 2026-02-22T22:01:22.042 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:01:22.042 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:22.042 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_34 2026-02-22T22:01:22.042 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T22:01:22.042 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:01:22.042 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T22:01:22.042 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T22:01:22.227 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.41524323749600567) > weight (0.25) 2026-02-22T22:01:22.227 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.9864952250419514) > weight (0.25) 2026-02-22T22:01:22.227 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.8981981051392511) > weight (0.25) 2026-02-22T22:01:22.227 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.3 with skip (0.45527897832064124) > weight (0.25) 2026-02-22T22:01:22.227 INFO:tasks.rbd_mirror_thrash:waiting for 70.3311925691636 secs before thrashing 2026-02-22T22:01:23.042 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 info mirror/image_34@snap 2026-02-22T22:01:23.069 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_34': 2026-02-22T22:01:23.069 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 128 MiB in 32 objects 2026-02-22T22:01:23.069 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T22:01:23.070 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 2 2026-02-22T22:01:23.070 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 1260d48d5a8f 2026-02-22T22:01:23.070 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.1260d48d5a8f 2026-02-22T22:01:23.070 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T22:01:23.070 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock, non-primary 2026-02-22T22:01:23.070 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T22:01:23.070 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T22:01:23.070 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:53:49 2026 2026-02-22T22:01:23.070 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:53:49 2026 2026-02-22T22:01:23.070 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:53:49 2026 2026-02-22T22:01:23.070 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: protected: False 2026-02-22T22:01:23.071 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T22:01:23.071 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T22:01:23.071 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: 787c83b7-1192-4f94-aa31-23be72fbd343 2026-02-22T22:01:23.071 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: false 2026-02-22T22:01:23.072 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:01:23.072 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ compare_image_snaps mirror image_34 snap 2026-02-22T22:01:23.072 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:23.072 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_34 2026-02-22T22:01:23.072 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T22:01:23.072 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local ret=0 2026-02-22T22:01:23.073 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local rmt_export=/tmp/tmp.j44LZPowal/cluster2-mirror-image_34.export 2026-02-22T22:01:23.073 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local loc_export=/tmp/tmp.j44LZPowal/cluster1-mirror-image_34.export 2026-02-22T22:01:23.073 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-image_34.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_34.export 2026-02-22T22:01:23.073 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror export image_34@snap /tmp/tmp.j44LZPowal/cluster2-mirror-image_34.export 2026-02-22T22:01:23.185 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:01:23.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 -p mirror export image_34@snap /tmp/tmp.j44LZPowal/cluster1-mirror-image_34.export 2026-02-22T22:01:23.310 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:01:23.317 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ cmp /tmp/tmp.j44LZPowal/cluster2-mirror-image_34.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_34.export 2026-02-22T22:01:23.366 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-image_34.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_34.export 2026-02-22T22:01:23.399 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:01:23.399 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:01:23.399 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_35 2026-02-22T22:01:23.400 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_snap cluster2 mirror image_35 snap 2026-02-22T22:01:23.400 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:01:23.400 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:23.400 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_35 2026-02-22T22:01:23.400 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T22:01:23.400 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror snap create image_35@snap --debug-rbd=20 --debug-journaler=20 2026-02-22T22:01:25.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_started cluster1 mirror image_35 2026-02-22T22:01:25.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:01:25.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:25.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_35 2026-02-22T22:01:25.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_state cluster1 mirror image_35 started 2026-02-22T22:01:25.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:01:25.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:25.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_35 2026-02-22T22:01:25.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=started 2026-02-22T22:01:25.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:01:25.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:01:25.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T22:01:26.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_replay_state cluster1 mirror image_35 started 2026-02-22T22:01:26.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:01:26.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:26.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_35 2026-02-22T22:01:26.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=started 2026-02-22T22:01:26.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_result 2026-02-22T22:01:26.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=stopped 2026-02-22T22:01:26.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_35 2026-02-22T22:01:26.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_instance=cluster1 2026-02-22T22:01:26.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:01:26.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ grep -i state 2026-02-22T22:01:26.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:01:26.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance=cluster1 2026-02-22T22:01:26.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local loop_instance 2026-02-22T22:01:26.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:01:26.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sleep 0 2026-02-22T22:01:26.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:01:26.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ seq 0 3 2026-02-22T22:01:26.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:01:26.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_35 2026-02-22T22:01:26.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:0 2026-02-22T22:01:26.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:01:26.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T22:01:26.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:01:26.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:0 2026-02-22T22:01:26.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T22:01:26.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T22:01:26.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:01:26.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=0 2026-02-22T22:01:26.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:01:26.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T22:01:26.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:01:26.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=0 2026-02-22T22:01:26.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T22:01:26.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:01:26.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:01:26.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T22:01:26.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T22:01:26.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:01:26.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:0 2026-02-22T22:01:26.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:01:26.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T22:01:26.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:01:26.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T22:01:26.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:01:26.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:01:26.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T22:01:26.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=0 2026-02-22T22:01:26.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=0 2026-02-22T22:01:26.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:01:26.215 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:01:26.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:01:26.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:01:26.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_35 2026-02-22T22:01:26.262 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_35'] 2026-02-22T22:01:26.262 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:01:26.263 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T22:01:26.266 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:01:26.266 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_35 2026-02-22T22:01:26.266 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:1 2026-02-22T22:01:26.266 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:01:26.266 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T22:01:26.266 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:01:26.266 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:1 2026-02-22T22:01:26.266 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T22:01:26.266 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T22:01:26.267 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:01:26.267 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=1 2026-02-22T22:01:26.267 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:01:26.267 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T22:01:26.267 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:01:26.267 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=1 2026-02-22T22:01:26.267 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T22:01:26.267 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:01:26.267 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:1 2026-02-22T22:01:26.267 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T22:01:26.267 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T22:01:26.267 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:01:26.268 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:1 2026-02-22T22:01:26.268 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:01:26.268 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T22:01:26.268 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:01:26.268 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T22:01:26.268 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T22:01:26.268 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:01:26.268 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T22:01:26.268 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=1 2026-02-22T22:01:26.268 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=1 2026-02-22T22:01:26.268 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:01:26.276 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:01:26.277 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:01:26.277 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:01:26.277 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_35 2026-02-22T22:01:26.322 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_35'] 2026-02-22T22:01:26.322 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:01:26.322 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T22:01:26.326 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:01:26.326 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_35 2026-02-22T22:01:26.326 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:2 2026-02-22T22:01:26.326 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:01:26.326 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T22:01:26.326 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:01:26.326 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:2 2026-02-22T22:01:26.326 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T22:01:26.327 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T22:01:26.327 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:01:26.327 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=2 2026-02-22T22:01:26.327 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:01:26.327 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T22:01:26.327 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:01:26.327 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=2 2026-02-22T22:01:26.327 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T22:01:26.327 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:01:26.327 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:2 2026-02-22T22:01:26.327 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T22:01:26.328 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T22:01:26.328 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:01:26.328 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:2 2026-02-22T22:01:26.328 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:01:26.328 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T22:01:26.328 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:01:26.328 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T22:01:26.328 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T22:01:26.328 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:01:26.328 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T22:01:26.328 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=2 2026-02-22T22:01:26.328 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=2 2026-02-22T22:01:26.329 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:01:26.336 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:01:26.336 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:01:26.336 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:01:26.336 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_35 2026-02-22T22:01:26.382 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_35'] 2026-02-22T22:01:26.382 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:01:26.382 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T22:01:26.385 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:01:26.386 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_35 2026-02-22T22:01:26.386 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:3 2026-02-22T22:01:26.386 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:01:26.386 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T22:01:26.386 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T22:01:26.386 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:3 2026-02-22T22:01:26.386 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T22:01:26.386 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T22:01:26.386 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:01:26.386 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=3 2026-02-22T22:01:26.386 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T22:01:26.387 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T22:01:26.387 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:01:26.387 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=3 2026-02-22T22:01:26.387 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=3 2026-02-22T22:01:26.387 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T22:01:26.387 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:3 2026-02-22T22:01:26.387 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T22:01:26.387 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T22:01:26.387 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T22:01:26.387 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:3 2026-02-22T22:01:26.387 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:01:26.387 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T22:01:26.388 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:01:26.388 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=3 2026-02-22T22:01:26.388 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T22:01:26.388 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:01:26.388 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T22:01:26.388 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=3 2026-02-22T22:01:26.388 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=3 2026-02-22T22:01:26.388 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T22:01:26.395 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:01:26.396 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:01:26.396 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:01:26.396 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_35 2026-02-22T22:01:26.449 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ return 0 2026-02-22T22:01:26.449 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:01:26.449 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:01:26.450 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep -i Replaying 2026-02-22T22:01:26.450 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: "state": "Replaying" 2026-02-22T22:01:26.453 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_state=started 2026-02-22T22:01:26.453 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test started = started 2026-02-22T22:01:26.453 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:01:26.453 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_35 2026-02-22T22:01:26.454 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T22:01:26.454 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:01:26.454 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:26.454 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_35 2026-02-22T22:01:26.454 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = journal ']' 2026-02-22T22:01:26.454 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T22:01:26.454 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ mirror_image_snapshot cluster2 mirror image_35 2026-02-22T22:01:26.454 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:01:26.454 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:26.455 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_35 2026-02-22T22:01:26.455 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_35 2026-02-22T22:01:27.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Snapshot ID: 154 2026-02-22T22:01:27.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_35 2026-02-22T22:01:27.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T22:01:27.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:01:27.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:27.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_35 2026-02-22T22:01:27.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster2-mirror-image_35.status 2026-02-22T22:01:27.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster2-mirror-image_35.status 2026-02-22T22:01:27.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:01:27.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_log=/tmp/tmp.j44LZPowal/cluster2-mirror-image_35.status 2026-02-22T22:01:27.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster1-mirror-image_35.status 2026-02-22T22:01:27.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster1-mirror-image_35.status 2026-02-22T22:01:27.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:01:27.209 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_status_log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_35.status 2026-02-22T22:01:27.209 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_35 /tmp/tmp.j44LZPowal/cluster2-mirror-image_35.status 2026-02-22T22:01:27.209 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:01:27.209 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:27.209 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_35 2026-02-22T22:01:27.209 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster2-mirror-image_35.status 2026-02-22T22:01:27.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_35 --format xml 2026-02-22T22:01:27.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:01:27.237 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T22:01:27.239 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snapshot_id=154 2026-02-22T22:01:27.239 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T22:01:27.239 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:01:27.239 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.2 2026-02-22T22:01:27.440 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_35 /tmp/tmp.j44LZPowal/cluster1-mirror-image_35.status 2026-02-22T22:01:27.441 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:01:27.441 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:27.441 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_35 2026-02-22T22:01:27.441 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_35.status 2026-02-22T22:01:27.441 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_35 --format xml 2026-02-22T22:01:27.441 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:01:27.471 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T22:01:27.473 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id=69 2026-02-22T22:01:27.473 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 154 = 69 2026-02-22T22:01:27.473 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:01:27.473 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.4 2026-02-22T22:01:27.874 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_35 /tmp/tmp.j44LZPowal/cluster1-mirror-image_35.status 2026-02-22T22:01:27.874 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:01:27.874 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:27.874 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_35 2026-02-22T22:01:27.875 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_35.status 2026-02-22T22:01:27.875 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_35 --format xml 2026-02-22T22:01:27.875 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:01:27.908 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T22:01:27.908 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T22:01:27.909 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T22:01:27.909 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T22:01:27.909 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T22:01:27.910 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T22:01:27.910 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 154 = '' 2026-02-22T22:01:27.910 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:01:27.910 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.8 2026-02-22T22:01:28.711 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_35 /tmp/tmp.j44LZPowal/cluster1-mirror-image_35.status 2026-02-22T22:01:28.711 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:01:28.711 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:28.711 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_35 2026-02-22T22:01:28.712 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_35.status 2026-02-22T22:01:28.712 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_35 --format xml 2026-02-22T22:01:28.712 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:01:28.742 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T22:01:28.744 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id=154 2026-02-22T22:01:28.744 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 154 = 154 2026-02-22T22:01:28.744 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:01:28.745 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snap_present cluster1 mirror image_35 snap 2026-02-22T22:01:28.745 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:01:28.745 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:28.745 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_35 2026-02-22T22:01:28.745 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T22:01:28.745 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:01:28.745 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T22:01:28.745 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T22:01:29.745 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 info mirror/image_35@snap 2026-02-22T22:01:29.773 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_35': 2026-02-22T22:01:29.773 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 128 MiB in 32 objects 2026-02-22T22:01:29.773 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T22:01:29.773 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 2 2026-02-22T22:01:29.773 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 1260e2f21c73 2026-02-22T22:01:29.773 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.1260e2f21c73 2026-02-22T22:01:29.773 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T22:01:29.773 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock, non-primary 2026-02-22T22:01:29.773 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T22:01:29.773 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T22:01:29.774 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:53:37 2026 2026-02-22T22:01:29.774 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:53:37 2026 2026-02-22T22:01:29.774 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:53:37 2026 2026-02-22T22:01:29.774 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: protected: False 2026-02-22T22:01:29.774 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T22:01:29.774 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T22:01:29.774 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: 7d8cf4a3-1fa1-4dcc-9d4d-2c1a44d0099a 2026-02-22T22:01:29.774 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: false 2026-02-22T22:01:29.775 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:01:29.776 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ compare_image_snaps mirror image_35 snap 2026-02-22T22:01:29.776 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:29.776 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_35 2026-02-22T22:01:29.776 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T22:01:29.776 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local ret=0 2026-02-22T22:01:29.776 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local rmt_export=/tmp/tmp.j44LZPowal/cluster2-mirror-image_35.export 2026-02-22T22:01:29.776 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local loc_export=/tmp/tmp.j44LZPowal/cluster1-mirror-image_35.export 2026-02-22T22:01:29.776 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-image_35.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_35.export 2026-02-22T22:01:29.776 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror export image_35@snap /tmp/tmp.j44LZPowal/cluster2-mirror-image_35.export 2026-02-22T22:01:29.890 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:01:29.897 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 -p mirror export image_35@snap /tmp/tmp.j44LZPowal/cluster1-mirror-image_35.export 2026-02-22T22:01:30.019 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:01:30.027 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ cmp /tmp/tmp.j44LZPowal/cluster2-mirror-image_35.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_35.export 2026-02-22T22:01:30.076 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-image_35.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_35.export 2026-02-22T22:01:30.112 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:01:30.112 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:01:30.112 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_36 2026-02-22T22:01:30.112 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_snap cluster2 mirror image_36 snap 2026-02-22T22:01:30.112 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:01:30.113 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:30.113 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_36 2026-02-22T22:01:30.113 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T22:01:30.113 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror snap create image_36@snap --debug-rbd=20 --debug-journaler=20 2026-02-22T22:01:31.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_started cluster1 mirror image_36 2026-02-22T22:01:31.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:01:31.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:31.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_36 2026-02-22T22:01:31.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_state cluster1 mirror image_36 started 2026-02-22T22:01:31.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:01:31.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:31.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_36 2026-02-22T22:01:31.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=started 2026-02-22T22:01:31.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:01:31.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:01:31.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T22:01:32.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_replay_state cluster1 mirror image_36 started 2026-02-22T22:01:32.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:01:32.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:32.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_36 2026-02-22T22:01:32.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=started 2026-02-22T22:01:32.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_result 2026-02-22T22:01:32.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=stopped 2026-02-22T22:01:32.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_36 2026-02-22T22:01:32.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_instance=cluster1 2026-02-22T22:01:32.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ grep -i state 2026-02-22T22:01:32.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:01:32.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:01:32.209 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance=cluster1 2026-02-22T22:01:32.209 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local loop_instance 2026-02-22T22:01:32.209 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:01:32.209 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sleep 0 2026-02-22T22:01:32.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:01:32.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ seq 0 3 2026-02-22T22:01:32.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:01:32.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_36 2026-02-22T22:01:32.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:0 2026-02-22T22:01:32.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:01:32.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T22:01:32.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:01:32.213 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:0 2026-02-22T22:01:32.213 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T22:01:32.213 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T22:01:32.213 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:01:32.213 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=0 2026-02-22T22:01:32.213 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:01:32.213 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T22:01:32.213 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:01:32.213 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=0 2026-02-22T22:01:32.213 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T22:01:32.213 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:01:32.213 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:01:32.213 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T22:01:32.214 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T22:01:32.214 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:01:32.214 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:0 2026-02-22T22:01:32.214 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:01:32.214 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T22:01:32.214 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:01:32.214 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T22:01:32.214 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:01:32.214 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:01:32.214 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T22:01:32.214 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=0 2026-02-22T22:01:32.214 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=0 2026-02-22T22:01:32.215 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:01:32.222 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:01:32.222 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:01:32.222 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:01:32.222 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_36 2026-02-22T22:01:32.267 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_36'] 2026-02-22T22:01:32.267 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:01:32.267 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T22:01:32.270 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:01:32.270 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_36 2026-02-22T22:01:32.270 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:1 2026-02-22T22:01:32.270 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:01:32.270 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T22:01:32.270 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:01:32.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:1 2026-02-22T22:01:32.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T22:01:32.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T22:01:32.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:01:32.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=1 2026-02-22T22:01:32.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:01:32.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T22:01:32.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:01:32.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=1 2026-02-22T22:01:32.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T22:01:32.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:01:32.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:1 2026-02-22T22:01:32.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T22:01:32.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T22:01:32.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:01:32.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:1 2026-02-22T22:01:32.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:01:32.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T22:01:32.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:01:32.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T22:01:32.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T22:01:32.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:01:32.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T22:01:32.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=1 2026-02-22T22:01:32.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=1 2026-02-22T22:01:32.273 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:01:32.281 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:01:32.281 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:01:32.281 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:01:32.282 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_36 2026-02-22T22:01:32.327 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_36'] 2026-02-22T22:01:32.327 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:01:32.327 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T22:01:32.330 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:01:32.330 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_36 2026-02-22T22:01:32.330 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:2 2026-02-22T22:01:32.330 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:01:32.330 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T22:01:32.330 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:01:32.330 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:2 2026-02-22T22:01:32.330 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T22:01:32.330 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T22:01:32.331 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:01:32.331 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=2 2026-02-22T22:01:32.331 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:01:32.331 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T22:01:32.331 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:01:32.331 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=2 2026-02-22T22:01:32.331 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T22:01:32.331 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:01:32.331 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:2 2026-02-22T22:01:32.331 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T22:01:32.331 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T22:01:32.331 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:01:32.332 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:2 2026-02-22T22:01:32.332 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:01:32.332 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T22:01:32.332 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:01:32.332 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T22:01:32.332 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T22:01:32.332 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:01:32.332 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T22:01:32.332 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=2 2026-02-22T22:01:32.332 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=2 2026-02-22T22:01:32.332 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:01:32.340 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:01:32.340 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:01:32.340 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:01:32.341 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_36 2026-02-22T22:01:32.386 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_36'] 2026-02-22T22:01:32.386 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:01:32.386 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T22:01:32.389 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:01:32.389 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_36 2026-02-22T22:01:32.389 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:3 2026-02-22T22:01:32.389 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:01:32.389 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T22:01:32.389 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T22:01:32.389 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:3 2026-02-22T22:01:32.389 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T22:01:32.389 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T22:01:32.390 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:01:32.390 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=3 2026-02-22T22:01:32.390 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T22:01:32.390 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T22:01:32.390 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:01:32.390 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=3 2026-02-22T22:01:32.390 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=3 2026-02-22T22:01:32.390 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T22:01:32.390 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:3 2026-02-22T22:01:32.390 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T22:01:32.390 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T22:01:32.390 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T22:01:32.390 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:3 2026-02-22T22:01:32.390 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:01:32.390 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T22:01:32.390 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:01:32.390 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=3 2026-02-22T22:01:32.391 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T22:01:32.391 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:01:32.391 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T22:01:32.391 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=3 2026-02-22T22:01:32.391 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=3 2026-02-22T22:01:32.391 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T22:01:32.399 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:01:32.400 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:01:32.400 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:01:32.400 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_36 2026-02-22T22:01:32.450 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ return 0 2026-02-22T22:01:32.451 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:01:32.451 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:01:32.451 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep -i Replaying 2026-02-22T22:01:32.452 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: "state": "Replaying" 2026-02-22T22:01:32.453 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_state=started 2026-02-22T22:01:32.453 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test started = started 2026-02-22T22:01:32.453 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:01:32.453 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_36 2026-02-22T22:01:32.453 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T22:01:32.453 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:01:32.453 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:32.453 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_36 2026-02-22T22:01:32.454 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = journal ']' 2026-02-22T22:01:32.454 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T22:01:32.454 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ mirror_image_snapshot cluster2 mirror image_36 2026-02-22T22:01:32.454 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:01:32.454 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:32.454 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_36 2026-02-22T22:01:32.454 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_36 2026-02-22T22:01:33.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Snapshot ID: 156 2026-02-22T22:01:33.213 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_36 2026-02-22T22:01:33.213 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T22:01:33.213 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:01:33.213 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:33.214 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_36 2026-02-22T22:01:33.214 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster2-mirror-image_36.status 2026-02-22T22:01:33.214 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster2-mirror-image_36.status 2026-02-22T22:01:33.214 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:01:33.214 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_log=/tmp/tmp.j44LZPowal/cluster2-mirror-image_36.status 2026-02-22T22:01:33.215 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster1-mirror-image_36.status 2026-02-22T22:01:33.215 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster1-mirror-image_36.status 2026-02-22T22:01:33.215 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:01:33.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_status_log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_36.status 2026-02-22T22:01:33.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_36 /tmp/tmp.j44LZPowal/cluster2-mirror-image_36.status 2026-02-22T22:01:33.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:01:33.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:33.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_36 2026-02-22T22:01:33.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster2-mirror-image_36.status 2026-02-22T22:01:33.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_36 --format xml 2026-02-22T22:01:33.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:01:33.244 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T22:01:33.245 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snapshot_id=156 2026-02-22T22:01:33.245 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T22:01:33.245 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:01:33.245 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.2 2026-02-22T22:01:33.447 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_36 /tmp/tmp.j44LZPowal/cluster1-mirror-image_36.status 2026-02-22T22:01:33.447 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:01:33.447 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:33.447 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_36 2026-02-22T22:01:33.447 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_36.status 2026-02-22T22:01:33.447 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_36 --format xml 2026-02-22T22:01:33.447 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:01:33.477 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T22:01:33.479 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id=70 2026-02-22T22:01:33.479 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 156 = 70 2026-02-22T22:01:33.479 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:01:33.479 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.4 2026-02-22T22:01:33.881 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_36 /tmp/tmp.j44LZPowal/cluster1-mirror-image_36.status 2026-02-22T22:01:33.881 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:01:33.881 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:33.881 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_36 2026-02-22T22:01:33.881 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_36.status 2026-02-22T22:01:33.881 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_36 --format xml 2026-02-22T22:01:33.881 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:01:33.912 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T22:01:33.912 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T22:01:33.913 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T22:01:33.913 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T22:01:33.913 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T22:01:33.914 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T22:01:33.914 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 156 = '' 2026-02-22T22:01:33.914 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:01:33.914 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.8 2026-02-22T22:01:34.715 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_36 /tmp/tmp.j44LZPowal/cluster1-mirror-image_36.status 2026-02-22T22:01:34.715 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:01:34.715 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:34.716 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_36 2026-02-22T22:01:34.716 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_36.status 2026-02-22T22:01:34.716 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_36 --format xml 2026-02-22T22:01:34.716 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:01:34.746 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T22:01:34.747 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id=156 2026-02-22T22:01:34.748 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 156 = 156 2026-02-22T22:01:34.748 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:01:34.748 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snap_present cluster1 mirror image_36 snap 2026-02-22T22:01:34.748 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:01:34.748 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:34.748 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_36 2026-02-22T22:01:34.748 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T22:01:34.748 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:01:34.748 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T22:01:34.748 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T22:01:35.748 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 info mirror/image_36@snap 2026-02-22T22:01:35.775 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_36': 2026-02-22T22:01:35.776 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 128 MiB in 32 objects 2026-02-22T22:01:35.776 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T22:01:35.776 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 2 2026-02-22T22:01:35.776 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 1260df68a4df 2026-02-22T22:01:35.776 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.1260df68a4df 2026-02-22T22:01:35.776 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T22:01:35.776 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock, non-primary 2026-02-22T22:01:35.776 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T22:01:35.776 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T22:01:35.777 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:53:45 2026 2026-02-22T22:01:35.777 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:53:45 2026 2026-02-22T22:01:35.777 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:53:45 2026 2026-02-22T22:01:35.777 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: protected: False 2026-02-22T22:01:35.777 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T22:01:35.777 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T22:01:35.777 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: 9860c66a-05af-466f-baca-3772f5e211ad 2026-02-22T22:01:35.777 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: false 2026-02-22T22:01:35.778 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:01:35.779 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ compare_image_snaps mirror image_36 snap 2026-02-22T22:01:35.779 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:35.779 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_36 2026-02-22T22:01:35.779 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T22:01:35.779 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local ret=0 2026-02-22T22:01:35.779 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local rmt_export=/tmp/tmp.j44LZPowal/cluster2-mirror-image_36.export 2026-02-22T22:01:35.779 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local loc_export=/tmp/tmp.j44LZPowal/cluster1-mirror-image_36.export 2026-02-22T22:01:35.779 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-image_36.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_36.export 2026-02-22T22:01:35.779 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror export image_36@snap /tmp/tmp.j44LZPowal/cluster2-mirror-image_36.export 2026-02-22T22:01:35.884 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:01:35.891 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 -p mirror export image_36@snap /tmp/tmp.j44LZPowal/cluster1-mirror-image_36.export 2026-02-22T22:01:36.016 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:01:36.023 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ cmp /tmp/tmp.j44LZPowal/cluster2-mirror-image_36.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_36.export 2026-02-22T22:01:36.072 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-image_36.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_36.export 2026-02-22T22:01:36.106 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:01:36.106 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:01:36.107 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_37 2026-02-22T22:01:36.107 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_snap cluster2 mirror image_37 snap 2026-02-22T22:01:36.107 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:01:36.107 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:36.107 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_37 2026-02-22T22:01:36.107 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T22:01:36.107 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror snap create image_37@snap --debug-rbd=20 --debug-journaler=20 2026-02-22T22:01:37.262 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_started cluster1 mirror image_37 2026-02-22T22:01:37.262 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:01:37.262 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:37.262 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_37 2026-02-22T22:01:37.262 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_state cluster1 mirror image_37 started 2026-02-22T22:01:37.262 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:01:37.262 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:37.262 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_37 2026-02-22T22:01:37.263 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=started 2026-02-22T22:01:37.263 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:01:37.263 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:01:37.263 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T22:01:38.263 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_replay_state cluster1 mirror image_37 started 2026-02-22T22:01:38.263 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:01:38.264 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:38.264 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_37 2026-02-22T22:01:38.264 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=started 2026-02-22T22:01:38.264 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_result 2026-02-22T22:01:38.264 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=stopped 2026-02-22T22:01:38.264 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_37 2026-02-22T22:01:38.264 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_instance=cluster1 2026-02-22T22:01:38.264 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:01:38.264 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:01:38.264 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ grep -i state 2026-02-22T22:01:38.265 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance=cluster1 2026-02-22T22:01:38.265 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local loop_instance 2026-02-22T22:01:38.265 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:01:38.265 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sleep 0 2026-02-22T22:01:38.268 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:01:38.268 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ seq 0 3 2026-02-22T22:01:38.268 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:01:38.268 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_37 2026-02-22T22:01:38.268 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:0 2026-02-22T22:01:38.268 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:01:38.269 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T22:01:38.269 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:01:38.269 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:0 2026-02-22T22:01:38.269 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T22:01:38.269 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T22:01:38.269 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:01:38.269 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=0 2026-02-22T22:01:38.269 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:01:38.269 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T22:01:38.269 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:01:38.269 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=0 2026-02-22T22:01:38.269 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T22:01:38.270 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:01:38.270 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:01:38.270 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T22:01:38.270 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T22:01:38.270 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:01:38.270 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:0 2026-02-22T22:01:38.270 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:01:38.270 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T22:01:38.270 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:01:38.270 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T22:01:38.270 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:01:38.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:01:38.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T22:01:38.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=0 2026-02-22T22:01:38.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=0 2026-02-22T22:01:38.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:01:38.279 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:01:38.279 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:01:38.280 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:01:38.280 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_37 2026-02-22T22:01:38.330 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ return 0 2026-02-22T22:01:38.331 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:01:38.331 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:01:38.332 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep -i Replaying 2026-02-22T22:01:38.332 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: "state": "Replaying" 2026-02-22T22:01:38.334 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_state=started 2026-02-22T22:01:38.334 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test started = started 2026-02-22T22:01:38.334 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:01:38.334 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_37 2026-02-22T22:01:38.334 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T22:01:38.334 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:01:38.334 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:38.334 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_37 2026-02-22T22:01:38.334 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = journal ']' 2026-02-22T22:01:38.334 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T22:01:38.335 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ mirror_image_snapshot cluster2 mirror image_37 2026-02-22T22:01:38.335 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:01:38.335 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:38.335 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_37 2026-02-22T22:01:38.335 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_37 2026-02-22T22:01:39.263 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Snapshot ID: 158 2026-02-22T22:01:39.269 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_37 2026-02-22T22:01:39.269 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T22:01:39.269 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:01:39.269 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:39.269 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_37 2026-02-22T22:01:39.269 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster2-mirror-image_37.status 2026-02-22T22:01:39.270 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster2-mirror-image_37.status 2026-02-22T22:01:39.270 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:01:39.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_log=/tmp/tmp.j44LZPowal/cluster2-mirror-image_37.status 2026-02-22T22:01:39.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster1-mirror-image_37.status 2026-02-22T22:01:39.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster1-mirror-image_37.status 2026-02-22T22:01:39.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:01:39.273 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_status_log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_37.status 2026-02-22T22:01:39.273 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_37 /tmp/tmp.j44LZPowal/cluster2-mirror-image_37.status 2026-02-22T22:01:39.273 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:01:39.273 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:39.273 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_37 2026-02-22T22:01:39.273 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster2-mirror-image_37.status 2026-02-22T22:01:39.273 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_37 --format xml 2026-02-22T22:01:39.273 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:01:39.300 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T22:01:39.301 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snapshot_id=158 2026-02-22T22:01:39.301 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T22:01:39.302 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:01:39.302 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.2 2026-02-22T22:01:39.503 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_37 /tmp/tmp.j44LZPowal/cluster1-mirror-image_37.status 2026-02-22T22:01:39.503 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:01:39.503 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:39.503 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_37 2026-02-22T22:01:39.503 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_37.status 2026-02-22T22:01:39.503 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_37 --format xml 2026-02-22T22:01:39.503 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:01:39.532 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T22:01:39.533 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T22:01:39.533 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T22:01:39.533 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T22:01:39.534 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T22:01:39.534 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T22:01:39.535 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 158 = '' 2026-02-22T22:01:39.535 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:01:39.535 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.4 2026-02-22T22:01:39.935 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_37 /tmp/tmp.j44LZPowal/cluster1-mirror-image_37.status 2026-02-22T22:01:39.935 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:01:39.935 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:39.935 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_37 2026-02-22T22:01:39.935 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_37.status 2026-02-22T22:01:39.935 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_37 --format xml 2026-02-22T22:01:39.935 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:01:39.963 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T22:01:39.964 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T22:01:39.964 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T22:01:39.965 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T22:01:39.965 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T22:01:39.965 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T22:01:39.966 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 158 = '' 2026-02-22T22:01:39.966 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:01:39.966 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.8 2026-02-22T22:01:40.766 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_37 /tmp/tmp.j44LZPowal/cluster1-mirror-image_37.status 2026-02-22T22:01:40.767 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:01:40.767 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:40.767 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_37 2026-02-22T22:01:40.767 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_37.status 2026-02-22T22:01:40.767 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_37 --format xml 2026-02-22T22:01:40.767 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:01:40.796 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T22:01:40.798 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id=158 2026-02-22T22:01:40.798 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 158 = 158 2026-02-22T22:01:40.798 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:01:40.798 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snap_present cluster1 mirror image_37 snap 2026-02-22T22:01:40.799 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:01:40.799 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:40.799 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_37 2026-02-22T22:01:40.799 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T22:01:40.799 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:01:40.799 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T22:01:40.799 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T22:01:41.799 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 info mirror/image_37@snap 2026-02-22T22:01:41.826 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_37': 2026-02-22T22:01:41.827 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 128 MiB in 32 objects 2026-02-22T22:01:41.827 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T22:01:41.827 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 2 2026-02-22T22:01:41.827 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 119a624749bd 2026-02-22T22:01:41.827 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.119a624749bd 2026-02-22T22:01:41.827 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T22:01:41.827 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock, non-primary 2026-02-22T22:01:41.827 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T22:01:41.827 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T22:01:41.827 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:53:57 2026 2026-02-22T22:01:41.828 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:53:57 2026 2026-02-22T22:01:41.828 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:53:57 2026 2026-02-22T22:01:41.828 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: protected: False 2026-02-22T22:01:41.828 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T22:01:41.828 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T22:01:41.828 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: 0ecb5a5c-9fb3-4659-bd3e-6eb17ad9c7a3 2026-02-22T22:01:41.828 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: false 2026-02-22T22:01:41.829 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:01:41.830 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ compare_image_snaps mirror image_37 snap 2026-02-22T22:01:41.830 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:41.830 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_37 2026-02-22T22:01:41.830 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T22:01:41.830 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local ret=0 2026-02-22T22:01:41.830 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local rmt_export=/tmp/tmp.j44LZPowal/cluster2-mirror-image_37.export 2026-02-22T22:01:41.830 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local loc_export=/tmp/tmp.j44LZPowal/cluster1-mirror-image_37.export 2026-02-22T22:01:41.830 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-image_37.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_37.export 2026-02-22T22:01:41.830 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror export image_37@snap /tmp/tmp.j44LZPowal/cluster2-mirror-image_37.export 2026-02-22T22:01:41.935 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:01:41.943 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 -p mirror export image_37@snap /tmp/tmp.j44LZPowal/cluster1-mirror-image_37.export 2026-02-22T22:01:42.061 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:01:42.069 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ cmp /tmp/tmp.j44LZPowal/cluster2-mirror-image_37.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_37.export 2026-02-22T22:01:42.120 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-image_37.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_37.export 2026-02-22T22:01:42.155 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:01:42.155 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:01:42.155 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_38 2026-02-22T22:01:42.155 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_snap cluster2 mirror image_38 snap 2026-02-22T22:01:42.155 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:01:42.155 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:42.155 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_38 2026-02-22T22:01:42.155 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T22:01:42.156 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror snap create image_38@snap --debug-rbd=20 --debug-journaler=20 2026-02-22T22:01:43.268 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_started cluster1 mirror image_38 2026-02-22T22:01:43.268 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:01:43.269 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:43.269 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_38 2026-02-22T22:01:43.269 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_state cluster1 mirror image_38 started 2026-02-22T22:01:43.269 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:01:43.269 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:43.269 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_38 2026-02-22T22:01:43.269 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=started 2026-02-22T22:01:43.269 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:01:43.269 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:01:43.269 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T22:01:43.781 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.36392294296299854) > weight (0.25) 2026-02-22T22:01:43.781 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.4250714666022477) > weight (0.25) 2026-02-22T22:01:43.782 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.2 2026-02-22T22:01:43.782 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Sent signal 15 2026-02-22T22:01:43.782 INFO:tasks.rbd_mirror_thrash:waiting for 1.0699555550176942 secs before reviving daemons 2026-02-22T22:01:43.783 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial026.stderr:2026-02-22T22:01:43.781+0000 7f8f83d5f640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2 (PID: 60959) UID: 1000 2026-02-22T22:01:44.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_replay_state cluster1 mirror image_38 started 2026-02-22T22:01:44.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:01:44.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:44.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_38 2026-02-22T22:01:44.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=started 2026-02-22T22:01:44.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_result 2026-02-22T22:01:44.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=stopped 2026-02-22T22:01:44.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_38 2026-02-22T22:01:44.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_instance=cluster1 2026-02-22T22:01:44.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:01:44.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:01:44.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance=cluster1 2026-02-22T22:01:44.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local loop_instance 2026-02-22T22:01:44.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ grep -i state 2026-02-22T22:01:44.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:01:44.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sleep 0 2026-02-22T22:01:44.273 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:01:44.275 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ seq 0 3 2026-02-22T22:01:44.275 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:01:44.276 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_38 2026-02-22T22:01:44.276 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:0 2026-02-22T22:01:44.276 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:01:44.276 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T22:01:44.276 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:01:44.276 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:0 2026-02-22T22:01:44.276 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T22:01:44.276 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T22:01:44.276 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:01:44.276 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=0 2026-02-22T22:01:44.276 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:01:44.277 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T22:01:44.277 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:01:44.277 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=0 2026-02-22T22:01:44.277 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T22:01:44.277 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:01:44.277 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:01:44.277 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T22:01:44.277 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T22:01:44.277 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:01:44.277 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:0 2026-02-22T22:01:44.277 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:01:44.277 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T22:01:44.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:01:44.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T22:01:44.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:01:44.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:01:44.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T22:01:44.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=0 2026-02-22T22:01:44.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=0 2026-02-22T22:01:44.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:01:44.287 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:01:44.287 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:01:44.287 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:01:44.287 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_38 2026-02-22T22:01:44.331 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_38'] 2026-02-22T22:01:44.332 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:01:44.332 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T22:01:44.335 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:01:44.335 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_38 2026-02-22T22:01:44.335 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:1 2026-02-22T22:01:44.335 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:01:44.335 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T22:01:44.335 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:01:44.335 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:1 2026-02-22T22:01:44.335 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T22:01:44.335 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T22:01:44.336 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:01:44.336 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=1 2026-02-22T22:01:44.336 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:01:44.336 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T22:01:44.336 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:01:44.336 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=1 2026-02-22T22:01:44.336 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T22:01:44.336 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:01:44.336 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:1 2026-02-22T22:01:44.336 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T22:01:44.336 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T22:01:44.336 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:01:44.337 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:1 2026-02-22T22:01:44.337 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:01:44.337 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T22:01:44.337 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:01:44.337 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T22:01:44.337 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T22:01:44.337 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:01:44.337 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T22:01:44.337 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=1 2026-02-22T22:01:44.337 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=1 2026-02-22T22:01:44.337 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:01:44.345 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:01:44.345 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:01:44.345 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:01:44.345 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_38 2026-02-22T22:01:44.390 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_38'] 2026-02-22T22:01:44.391 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:01:44.391 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T22:01:44.394 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:01:44.394 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_38 2026-02-22T22:01:44.394 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:2 2026-02-22T22:01:44.394 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:01:44.394 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T22:01:44.394 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:01:44.395 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:2 2026-02-22T22:01:44.395 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T22:01:44.395 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T22:01:44.395 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:01:44.395 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=2 2026-02-22T22:01:44.395 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:01:44.395 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T22:01:44.395 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:01:44.395 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=2 2026-02-22T22:01:44.395 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T22:01:44.395 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:01:44.396 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:2 2026-02-22T22:01:44.396 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T22:01:44.396 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T22:01:44.396 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:01:44.396 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:2 2026-02-22T22:01:44.396 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:01:44.396 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T22:01:44.396 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:01:44.396 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T22:01:44.396 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T22:01:44.397 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:01:44.397 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T22:01:44.397 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=2 2026-02-22T22:01:44.397 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=2 2026-02-22T22:01:44.397 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:01:44.404 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:01:44.404 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:01:44.404 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:01:44.404 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_38 2026-02-22T22:01:44.453 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ return 0 2026-02-22T22:01:44.454 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:01:44.454 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:01:44.454 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep -i Replaying 2026-02-22T22:01:44.455 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: "state": "Replaying" 2026-02-22T22:01:44.456 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_state=started 2026-02-22T22:01:44.457 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test started = started 2026-02-22T22:01:44.457 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:01:44.457 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_38 2026-02-22T22:01:44.457 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T22:01:44.457 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:01:44.457 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:44.457 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_38 2026-02-22T22:01:44.457 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = journal ']' 2026-02-22T22:01:44.457 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T22:01:44.457 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ mirror_image_snapshot cluster2 mirror image_38 2026-02-22T22:01:44.458 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:01:44.458 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:44.458 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_38 2026-02-22T22:01:44.458 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_38 2026-02-22T22:01:44.853 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.2 2026-02-22T22:01:44.853 INFO:teuthology.orchestra.run:waiting for 600 2026-02-22T22:01:44.854 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.2 2026-02-22T22:01:44.854 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Restarting daemon 2026-02-22T22:01:44.854 DEBUG:teuthology.orchestra.run.trial026:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2 2026-02-22T22:01:44.857 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Started 2026-02-22T22:01:44.857 INFO:tasks.rbd_mirror_thrash:waiting for 110.52519882011958 secs before thrashing 2026-02-22T22:01:45.273 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Snapshot ID: 160 2026-02-22T22:01:45.280 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_38 2026-02-22T22:01:45.280 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T22:01:45.280 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:01:45.280 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:45.281 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_38 2026-02-22T22:01:45.281 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster2-mirror-image_38.status 2026-02-22T22:01:45.281 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster2-mirror-image_38.status 2026-02-22T22:01:45.281 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:01:45.281 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_log=/tmp/tmp.j44LZPowal/cluster2-mirror-image_38.status 2026-02-22T22:01:45.282 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster1-mirror-image_38.status 2026-02-22T22:01:45.282 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster1-mirror-image_38.status 2026-02-22T22:01:45.282 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:01:45.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_status_log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_38.status 2026-02-22T22:01:45.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_38 /tmp/tmp.j44LZPowal/cluster2-mirror-image_38.status 2026-02-22T22:01:45.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:01:45.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:45.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_38 2026-02-22T22:01:45.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster2-mirror-image_38.status 2026-02-22T22:01:45.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_38 --format xml 2026-02-22T22:01:45.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:01:45.311 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T22:01:45.313 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snapshot_id=160 2026-02-22T22:01:45.313 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T22:01:45.313 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:01:45.313 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.2 2026-02-22T22:01:45.514 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_38 /tmp/tmp.j44LZPowal/cluster1-mirror-image_38.status 2026-02-22T22:01:45.514 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:01:45.515 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:45.515 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_38 2026-02-22T22:01:45.515 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_38.status 2026-02-22T22:01:45.515 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_38 --format xml 2026-02-22T22:01:45.515 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:01:45.545 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T22:01:45.545 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T22:01:45.546 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T22:01:45.546 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T22:01:45.546 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T22:01:45.547 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T22:01:45.547 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 160 = '' 2026-02-22T22:01:45.547 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:01:45.547 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.4 2026-02-22T22:01:45.948 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_38 /tmp/tmp.j44LZPowal/cluster1-mirror-image_38.status 2026-02-22T22:01:45.948 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:01:45.949 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:45.949 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_38 2026-02-22T22:01:45.949 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_38.status 2026-02-22T22:01:45.949 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_38 --format xml 2026-02-22T22:01:45.949 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:01:45.978 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T22:01:45.978 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T22:01:45.979 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T22:01:45.979 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T22:01:45.979 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T22:01:45.980 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T22:01:45.980 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 160 = '' 2026-02-22T22:01:45.980 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:01:45.980 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.8 2026-02-22T22:01:46.781 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_38 /tmp/tmp.j44LZPowal/cluster1-mirror-image_38.status 2026-02-22T22:01:46.781 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:01:46.781 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:46.781 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_38 2026-02-22T22:01:46.782 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_38.status 2026-02-22T22:01:46.782 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_38 --format xml 2026-02-22T22:01:46.782 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:01:46.812 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T22:01:46.814 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id=160 2026-02-22T22:01:46.814 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 160 = 160 2026-02-22T22:01:46.815 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:01:46.815 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snap_present cluster1 mirror image_38 snap 2026-02-22T22:01:46.815 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:01:46.815 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:46.815 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_38 2026-02-22T22:01:46.815 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T22:01:46.815 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:01:46.816 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T22:01:46.816 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T22:01:47.814 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 info mirror/image_38@snap 2026-02-22T22:01:47.842 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_38': 2026-02-22T22:01:47.842 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 128 MiB in 32 objects 2026-02-22T22:01:47.842 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T22:01:47.842 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 2 2026-02-22T22:01:47.843 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 1227a2d20dfb 2026-02-22T22:01:47.843 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.1227a2d20dfb 2026-02-22T22:01:47.843 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T22:01:47.843 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock, non-primary 2026-02-22T22:01:47.843 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T22:01:47.843 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T22:01:47.843 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:54:12 2026 2026-02-22T22:01:47.843 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:54:12 2026 2026-02-22T22:01:47.843 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:54:12 2026 2026-02-22T22:01:47.843 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: protected: False 2026-02-22T22:01:47.844 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T22:01:47.844 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T22:01:47.844 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: db20ecda-1e60-4b4e-b471-d368349418c1 2026-02-22T22:01:47.844 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: false 2026-02-22T22:01:47.845 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:01:47.845 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ compare_image_snaps mirror image_38 snap 2026-02-22T22:01:47.845 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:47.845 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_38 2026-02-22T22:01:47.845 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T22:01:47.845 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local ret=0 2026-02-22T22:01:47.845 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local rmt_export=/tmp/tmp.j44LZPowal/cluster2-mirror-image_38.export 2026-02-22T22:01:47.846 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local loc_export=/tmp/tmp.j44LZPowal/cluster1-mirror-image_38.export 2026-02-22T22:01:47.846 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-image_38.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_38.export 2026-02-22T22:01:47.846 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror export image_38@snap /tmp/tmp.j44LZPowal/cluster2-mirror-image_38.export 2026-02-22T22:01:47.949 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:01:47.956 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 -p mirror export image_38@snap /tmp/tmp.j44LZPowal/cluster1-mirror-image_38.export 2026-02-22T22:01:48.079 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:01:48.087 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ cmp /tmp/tmp.j44LZPowal/cluster2-mirror-image_38.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_38.export 2026-02-22T22:01:48.137 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-image_38.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_38.export 2026-02-22T22:01:48.173 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:01:48.173 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:01:48.173 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_39 2026-02-22T22:01:48.173 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_snap cluster2 mirror image_39 snap 2026-02-22T22:01:48.173 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:01:48.173 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:48.174 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_39 2026-02-22T22:01:48.174 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T22:01:48.174 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror snap create image_39@snap --debug-rbd=20 --debug-journaler=20 2026-02-22T22:01:49.283 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_started cluster1 mirror image_39 2026-02-22T22:01:49.283 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:01:49.283 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:49.283 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_39 2026-02-22T22:01:49.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_state cluster1 mirror image_39 started 2026-02-22T22:01:49.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:01:49.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:49.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_39 2026-02-22T22:01:49.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=started 2026-02-22T22:01:49.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:01:49.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:01:49.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T22:01:50.285 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_replay_state cluster1 mirror image_39 started 2026-02-22T22:01:50.285 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:01:50.285 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:50.285 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_39 2026-02-22T22:01:50.285 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=started 2026-02-22T22:01:50.285 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_result 2026-02-22T22:01:50.285 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=stopped 2026-02-22T22:01:50.285 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_39 2026-02-22T22:01:50.285 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_instance=cluster1 2026-02-22T22:01:50.286 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:01:50.286 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:01:50.286 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance=cluster1 2026-02-22T22:01:50.286 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local loop_instance 2026-02-22T22:01:50.286 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:01:50.286 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sleep 0 2026-02-22T22:01:50.286 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ grep -i state 2026-02-22T22:01:50.289 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:01:50.289 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ seq 0 3 2026-02-22T22:01:50.289 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:01:50.289 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_39 2026-02-22T22:01:50.289 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:0 2026-02-22T22:01:50.289 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:01:50.290 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T22:01:50.290 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:01:50.290 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:0 2026-02-22T22:01:50.290 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T22:01:50.290 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T22:01:50.290 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:01:50.290 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=0 2026-02-22T22:01:50.290 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:01:50.290 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T22:01:50.290 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:01:50.290 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=0 2026-02-22T22:01:50.290 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T22:01:50.291 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:01:50.291 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:01:50.291 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T22:01:50.291 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T22:01:50.291 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:01:50.291 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:0 2026-02-22T22:01:50.291 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:01:50.291 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T22:01:50.291 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:01:50.291 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T22:01:50.291 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:01:50.291 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:01:50.291 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T22:01:50.292 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=0 2026-02-22T22:01:50.292 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=0 2026-02-22T22:01:50.292 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:01:50.300 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:01:50.300 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:01:50.301 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:01:50.301 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_39 2026-02-22T22:01:50.345 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_39'] 2026-02-22T22:01:50.345 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:01:50.345 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T22:01:50.347 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:01:50.347 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_39 2026-02-22T22:01:50.347 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:1 2026-02-22T22:01:50.347 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:01:50.347 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T22:01:50.347 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:01:50.347 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:1 2026-02-22T22:01:50.347 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T22:01:50.347 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T22:01:50.347 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:01:50.347 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=1 2026-02-22T22:01:50.347 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:01:50.347 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T22:01:50.347 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:01:50.347 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=1 2026-02-22T22:01:50.347 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T22:01:50.347 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:01:50.347 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:1 2026-02-22T22:01:50.347 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T22:01:50.347 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T22:01:50.347 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:01:50.347 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:1 2026-02-22T22:01:50.347 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:01:50.347 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T22:01:50.347 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:01:50.347 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T22:01:50.347 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T22:01:50.347 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:01:50.347 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T22:01:50.347 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=1 2026-02-22T22:01:50.348 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=1 2026-02-22T22:01:50.348 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:01:50.359 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:01:50.360 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:01:50.360 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:01:50.360 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_39 2026-02-22T22:01:50.412 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ return 0 2026-02-22T22:01:50.413 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:01:50.413 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:01:50.413 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep -i Replaying 2026-02-22T22:01:50.414 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: "state": "Replaying" 2026-02-22T22:01:50.415 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_state=started 2026-02-22T22:01:50.415 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test started = started 2026-02-22T22:01:50.415 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:01:50.415 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_39 2026-02-22T22:01:50.415 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T22:01:50.415 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:01:50.415 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:50.416 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_39 2026-02-22T22:01:50.416 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = journal ']' 2026-02-22T22:01:50.416 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T22:01:50.416 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ mirror_image_snapshot cluster2 mirror image_39 2026-02-22T22:01:50.416 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:01:50.416 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:50.416 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_39 2026-02-22T22:01:50.416 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_39 2026-02-22T22:01:51.285 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Snapshot ID: 162 2026-02-22T22:01:51.292 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_39 2026-02-22T22:01:51.292 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T22:01:51.292 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:01:51.292 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:51.292 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_39 2026-02-22T22:01:51.292 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster2-mirror-image_39.status 2026-02-22T22:01:51.293 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster2-mirror-image_39.status 2026-02-22T22:01:51.293 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:01:51.293 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_log=/tmp/tmp.j44LZPowal/cluster2-mirror-image_39.status 2026-02-22T22:01:51.294 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster1-mirror-image_39.status 2026-02-22T22:01:51.294 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster1-mirror-image_39.status 2026-02-22T22:01:51.294 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:01:51.296 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_status_log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_39.status 2026-02-22T22:01:51.296 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_39 /tmp/tmp.j44LZPowal/cluster2-mirror-image_39.status 2026-02-22T22:01:51.296 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:01:51.296 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:51.296 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_39 2026-02-22T22:01:51.296 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster2-mirror-image_39.status 2026-02-22T22:01:51.296 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_39 --format xml 2026-02-22T22:01:51.297 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:01:51.324 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T22:01:51.325 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snapshot_id=162 2026-02-22T22:01:51.325 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T22:01:51.325 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:01:51.325 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.2 2026-02-22T22:01:51.527 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_39 /tmp/tmp.j44LZPowal/cluster1-mirror-image_39.status 2026-02-22T22:01:51.527 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:01:51.527 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:51.527 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_39 2026-02-22T22:01:51.528 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_39.status 2026-02-22T22:01:51.528 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_39 --format xml 2026-02-22T22:01:51.528 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:01:51.557 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T22:01:51.557 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T22:01:51.558 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T22:01:51.558 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T22:01:51.558 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T22:01:51.559 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T22:01:51.559 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 162 = '' 2026-02-22T22:01:51.559 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:01:51.559 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.4 2026-02-22T22:01:51.959 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_39 /tmp/tmp.j44LZPowal/cluster1-mirror-image_39.status 2026-02-22T22:01:51.959 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:01:51.960 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:51.960 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_39 2026-02-22T22:01:51.960 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_39.status 2026-02-22T22:01:51.960 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_39 --format xml 2026-02-22T22:01:51.960 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:01:51.989 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T22:01:51.989 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T22:01:51.990 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T22:01:51.990 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T22:01:51.990 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T22:01:51.991 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T22:01:51.991 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 162 = '' 2026-02-22T22:01:51.991 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:01:51.991 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.8 2026-02-22T22:01:52.792 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_39 /tmp/tmp.j44LZPowal/cluster1-mirror-image_39.status 2026-02-22T22:01:52.792 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:01:52.793 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:52.793 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_39 2026-02-22T22:01:52.793 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_39.status 2026-02-22T22:01:52.793 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_39 --format xml 2026-02-22T22:01:52.793 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:01:52.824 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T22:01:52.826 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id=162 2026-02-22T22:01:52.826 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 162 = 162 2026-02-22T22:01:52.827 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:01:52.827 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snap_present cluster1 mirror image_39 snap 2026-02-22T22:01:52.827 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:01:52.827 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:52.827 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_39 2026-02-22T22:01:52.827 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T22:01:52.827 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:01:52.827 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T22:01:52.827 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T22:01:53.827 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 info mirror/image_39@snap 2026-02-22T22:01:53.857 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_39': 2026-02-22T22:01:53.857 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 128 MiB in 32 objects 2026-02-22T22:01:53.857 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T22:01:53.857 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 2 2026-02-22T22:01:53.857 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 1248ed578f35 2026-02-22T22:01:53.857 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.1248ed578f35 2026-02-22T22:01:53.857 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T22:01:53.857 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock, non-primary 2026-02-22T22:01:53.857 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T22:01:53.857 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T22:01:53.857 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:54:22 2026 2026-02-22T22:01:53.858 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:54:22 2026 2026-02-22T22:01:53.858 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:54:22 2026 2026-02-22T22:01:53.858 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: protected: False 2026-02-22T22:01:53.858 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T22:01:53.858 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T22:01:53.858 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: 92432590-5c9b-4a0c-b269-4ed3db4a7657 2026-02-22T22:01:53.858 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: false 2026-02-22T22:01:53.859 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:01:53.859 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ compare_image_snaps mirror image_39 snap 2026-02-22T22:01:53.859 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:53.859 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_39 2026-02-22T22:01:53.859 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T22:01:53.859 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local ret=0 2026-02-22T22:01:53.859 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local rmt_export=/tmp/tmp.j44LZPowal/cluster2-mirror-image_39.export 2026-02-22T22:01:53.860 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local loc_export=/tmp/tmp.j44LZPowal/cluster1-mirror-image_39.export 2026-02-22T22:01:53.860 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-image_39.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_39.export 2026-02-22T22:01:53.860 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror export image_39@snap /tmp/tmp.j44LZPowal/cluster2-mirror-image_39.export 2026-02-22T22:01:53.966 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:01:53.974 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 -p mirror export image_39@snap /tmp/tmp.j44LZPowal/cluster1-mirror-image_39.export 2026-02-22T22:01:54.106 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:01:54.113 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ cmp /tmp/tmp.j44LZPowal/cluster2-mirror-image_39.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_39.export 2026-02-22T22:01:54.163 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-image_39.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_39.export 2026-02-22T22:01:54.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:01:54.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:01:54.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_40 2026-02-22T22:01:54.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_snap cluster2 mirror image_40 snap 2026-02-22T22:01:54.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:01:54.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:54.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_40 2026-02-22T22:01:54.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T22:01:54.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror snap create image_40@snap --debug-rbd=20 --debug-journaler=20 2026-02-22T22:01:55.291 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_started cluster1 mirror image_40 2026-02-22T22:01:55.291 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:01:55.291 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:55.292 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_40 2026-02-22T22:01:55.292 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_state cluster1 mirror image_40 started 2026-02-22T22:01:55.292 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:01:55.292 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:55.292 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_40 2026-02-22T22:01:55.292 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=started 2026-02-22T22:01:55.292 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:01:55.292 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:01:55.292 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T22:01:56.293 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_replay_state cluster1 mirror image_40 started 2026-02-22T22:01:56.294 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:01:56.294 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:56.294 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_40 2026-02-22T22:01:56.294 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=started 2026-02-22T22:01:56.294 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_result 2026-02-22T22:01:56.294 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=stopped 2026-02-22T22:01:56.294 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_40 2026-02-22T22:01:56.294 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_instance=cluster1 2026-02-22T22:01:56.294 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ grep -i state 2026-02-22T22:01:56.295 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:01:56.295 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:01:56.295 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance=cluster1 2026-02-22T22:01:56.295 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local loop_instance 2026-02-22T22:01:56.295 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:01:56.295 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sleep 0 2026-02-22T22:01:56.298 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:01:56.298 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ seq 0 3 2026-02-22T22:01:56.299 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:01:56.299 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_40 2026-02-22T22:01:56.299 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:0 2026-02-22T22:01:56.299 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:01:56.299 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T22:01:56.299 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:01:56.299 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:0 2026-02-22T22:01:56.299 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T22:01:56.299 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T22:01:56.299 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:01:56.299 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=0 2026-02-22T22:01:56.300 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:01:56.300 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T22:01:56.300 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:01:56.300 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=0 2026-02-22T22:01:56.300 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T22:01:56.300 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:01:56.300 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:01:56.300 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T22:01:56.300 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T22:01:56.300 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:01:56.300 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:0 2026-02-22T22:01:56.300 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:01:56.301 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T22:01:56.301 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:01:56.301 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T22:01:56.301 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:01:56.301 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:01:56.301 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T22:01:56.301 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=0 2026-02-22T22:01:56.301 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=0 2026-02-22T22:01:56.301 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:01:56.309 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:01:56.309 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:01:56.309 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:01:56.309 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_40 2026-02-22T22:01:56.355 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_40'] 2026-02-22T22:01:56.355 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:01:56.355 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T22:01:56.359 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:01:56.359 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_40 2026-02-22T22:01:56.360 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:1 2026-02-22T22:01:56.360 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:01:56.360 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T22:01:56.360 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:01:56.360 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:1 2026-02-22T22:01:56.360 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T22:01:56.360 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T22:01:56.360 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:01:56.360 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=1 2026-02-22T22:01:56.360 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:01:56.360 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T22:01:56.361 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:01:56.361 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=1 2026-02-22T22:01:56.361 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T22:01:56.361 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:01:56.361 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:1 2026-02-22T22:01:56.361 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T22:01:56.361 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T22:01:56.361 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:01:56.361 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:1 2026-02-22T22:01:56.361 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:01:56.361 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T22:01:56.361 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:01:56.362 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T22:01:56.362 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T22:01:56.362 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:01:56.362 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T22:01:56.362 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=1 2026-02-22T22:01:56.362 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=1 2026-02-22T22:01:56.363 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:01:56.369 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:01:56.369 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:01:56.369 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:01:56.369 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_40 2026-02-22T22:01:56.414 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_40'] 2026-02-22T22:01:56.414 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:01:56.414 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T22:01:56.420 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:01:56.420 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_40 2026-02-22T22:01:56.421 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:2 2026-02-22T22:01:56.421 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:01:56.421 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T22:01:56.421 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:01:56.421 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:2 2026-02-22T22:01:56.421 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T22:01:56.421 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T22:01:56.421 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:01:56.421 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=2 2026-02-22T22:01:56.422 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:01:56.422 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T22:01:56.422 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:01:56.422 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=2 2026-02-22T22:01:56.422 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T22:01:56.422 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:01:56.422 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:2 2026-02-22T22:01:56.422 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T22:01:56.422 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T22:01:56.422 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:01:56.422 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:2 2026-02-22T22:01:56.423 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:01:56.423 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T22:01:56.423 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:01:56.423 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T22:01:56.423 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T22:01:56.423 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:01:56.423 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T22:01:56.423 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=2 2026-02-22T22:01:56.424 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=2 2026-02-22T22:01:56.424 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:01:56.427 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:01:56.427 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:01:56.427 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:01:56.428 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_40 2026-02-22T22:01:56.473 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_40'] 2026-02-22T22:01:56.473 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:01:56.473 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T22:01:56.476 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:01:56.476 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_40 2026-02-22T22:01:56.476 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:3 2026-02-22T22:01:56.476 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:01:56.476 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T22:01:56.477 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T22:01:56.477 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:3 2026-02-22T22:01:56.477 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T22:01:56.477 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T22:01:56.477 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:01:56.477 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=3 2026-02-22T22:01:56.477 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T22:01:56.477 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T22:01:56.477 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:01:56.477 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=3 2026-02-22T22:01:56.477 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=3 2026-02-22T22:01:56.477 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T22:01:56.478 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:3 2026-02-22T22:01:56.478 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T22:01:56.478 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T22:01:56.478 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T22:01:56.478 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:3 2026-02-22T22:01:56.478 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:01:56.478 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T22:01:56.478 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:01:56.478 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=3 2026-02-22T22:01:56.478 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T22:01:56.478 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:01:56.478 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T22:01:56.479 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=3 2026-02-22T22:01:56.479 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=3 2026-02-22T22:01:56.479 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T22:01:56.486 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:01:56.486 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:01:56.486 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:01:56.486 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_40 2026-02-22T22:01:56.535 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ return 0 2026-02-22T22:01:56.536 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:01:56.536 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:01:56.536 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep -i Replaying 2026-02-22T22:01:56.536 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: "state": "Replaying" 2026-02-22T22:01:56.538 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_state=started 2026-02-22T22:01:56.538 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test started = started 2026-02-22T22:01:56.538 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:01:56.538 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_40 2026-02-22T22:01:56.538 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T22:01:56.538 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:01:56.538 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:56.538 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_40 2026-02-22T22:01:56.538 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = journal ']' 2026-02-22T22:01:56.538 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T22:01:56.539 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ mirror_image_snapshot cluster2 mirror image_40 2026-02-22T22:01:56.539 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:01:56.539 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:56.539 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_40 2026-02-22T22:01:56.539 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_40 2026-02-22T22:01:57.295 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Snapshot ID: 164 2026-02-22T22:01:57.301 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_40 2026-02-22T22:01:57.301 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T22:01:57.301 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:01:57.302 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:57.302 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_40 2026-02-22T22:01:57.302 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster2-mirror-image_40.status 2026-02-22T22:01:57.302 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster2-mirror-image_40.status 2026-02-22T22:01:57.302 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:01:57.303 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_log=/tmp/tmp.j44LZPowal/cluster2-mirror-image_40.status 2026-02-22T22:01:57.303 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster1-mirror-image_40.status 2026-02-22T22:01:57.304 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster1-mirror-image_40.status 2026-02-22T22:01:57.304 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:01:57.305 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_status_log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_40.status 2026-02-22T22:01:57.305 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_40 /tmp/tmp.j44LZPowal/cluster2-mirror-image_40.status 2026-02-22T22:01:57.306 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:01:57.306 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:57.306 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_40 2026-02-22T22:01:57.306 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster2-mirror-image_40.status 2026-02-22T22:01:57.306 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_40 --format xml 2026-02-22T22:01:57.306 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:01:57.333 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T22:01:57.335 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snapshot_id=164 2026-02-22T22:01:57.335 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T22:01:57.335 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:01:57.335 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.2 2026-02-22T22:01:57.535 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_40 /tmp/tmp.j44LZPowal/cluster1-mirror-image_40.status 2026-02-22T22:01:57.536 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:01:57.536 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:57.536 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_40 2026-02-22T22:01:57.536 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_40.status 2026-02-22T22:01:57.536 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_40 --format xml 2026-02-22T22:01:57.536 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:01:57.565 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T22:01:57.565 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T22:01:57.566 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T22:01:57.566 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T22:01:57.566 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T22:01:57.566 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T22:01:57.567 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 164 = '' 2026-02-22T22:01:57.567 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:01:57.567 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.4 2026-02-22T22:01:57.968 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_40 /tmp/tmp.j44LZPowal/cluster1-mirror-image_40.status 2026-02-22T22:01:57.969 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:01:57.969 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:57.969 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_40 2026-02-22T22:01:57.969 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_40.status 2026-02-22T22:01:57.969 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_40 --format xml 2026-02-22T22:01:57.969 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:01:57.998 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T22:01:57.998 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T22:01:57.999 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T22:01:57.999 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T22:01:57.999 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T22:01:58.000 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T22:01:58.000 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 164 = '' 2026-02-22T22:01:58.000 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:01:58.000 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.8 2026-02-22T22:01:58.800 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_40 /tmp/tmp.j44LZPowal/cluster1-mirror-image_40.status 2026-02-22T22:01:58.801 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:01:58.801 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:58.801 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_40 2026-02-22T22:01:58.801 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_40.status 2026-02-22T22:01:58.801 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_40 --format xml 2026-02-22T22:01:58.801 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:01:58.831 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T22:01:58.834 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id=164 2026-02-22T22:01:58.834 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 164 = 164 2026-02-22T22:01:58.834 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:01:58.834 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snap_present cluster1 mirror image_40 snap 2026-02-22T22:01:58.834 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:01:58.834 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:58.835 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_40 2026-02-22T22:01:58.835 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T22:01:58.835 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:01:58.835 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T22:01:58.835 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T22:01:59.834 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 info mirror/image_40@snap 2026-02-22T22:01:59.862 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_40': 2026-02-22T22:01:59.862 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 128 MiB in 32 objects 2026-02-22T22:01:59.862 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T22:01:59.862 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 2 2026-02-22T22:01:59.862 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 1260e4433085 2026-02-22T22:01:59.862 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.1260e4433085 2026-02-22T22:01:59.863 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T22:01:59.863 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock, non-primary 2026-02-22T22:01:59.863 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T22:01:59.863 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T22:01:59.863 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:54:30 2026 2026-02-22T22:01:59.863 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:54:30 2026 2026-02-22T22:01:59.863 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:54:30 2026 2026-02-22T22:01:59.863 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: protected: False 2026-02-22T22:01:59.863 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T22:01:59.863 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T22:01:59.863 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: c57ae3a7-f4fd-4810-a324-ac5f588ec54e 2026-02-22T22:01:59.864 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: false 2026-02-22T22:01:59.865 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:01:59.865 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ compare_image_snaps mirror image_40 snap 2026-02-22T22:01:59.865 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:01:59.865 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_40 2026-02-22T22:01:59.865 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T22:01:59.865 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local ret=0 2026-02-22T22:01:59.865 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local rmt_export=/tmp/tmp.j44LZPowal/cluster2-mirror-image_40.export 2026-02-22T22:01:59.866 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local loc_export=/tmp/tmp.j44LZPowal/cluster1-mirror-image_40.export 2026-02-22T22:01:59.866 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-image_40.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_40.export 2026-02-22T22:01:59.866 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror export image_40@snap /tmp/tmp.j44LZPowal/cluster2-mirror-image_40.export 2026-02-22T22:02:00.171 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:02:00.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 -p mirror export image_40@snap /tmp/tmp.j44LZPowal/cluster1-mirror-image_40.export 2026-02-22T22:02:00.303 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:02:00.311 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ cmp /tmp/tmp.j44LZPowal/cluster2-mirror-image_40.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_40.export 2026-02-22T22:02:00.361 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-image_40.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_40.export 2026-02-22T22:02:00.394 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:02:00.394 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:02:00.394 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_41 2026-02-22T22:02:00.394 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_snap cluster2 mirror image_41 snap 2026-02-22T22:02:00.394 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:02:00.394 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:02:00.394 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_41 2026-02-22T22:02:00.394 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T22:02:00.394 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror snap create image_41@snap --debug-rbd=20 --debug-journaler=20 2026-02-22T22:02:01.492 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_started cluster1 mirror image_41 2026-02-22T22:02:01.492 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:02:01.492 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:02:01.492 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_41 2026-02-22T22:02:01.492 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_state cluster1 mirror image_41 started 2026-02-22T22:02:01.493 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:02:01.493 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:02:01.493 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_41 2026-02-22T22:02:01.493 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=started 2026-02-22T22:02:01.493 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:02:01.493 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:02:01.493 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T22:02:02.493 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_replay_state cluster1 mirror image_41 started 2026-02-22T22:02:02.494 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:02:02.494 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:02:02.494 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_41 2026-02-22T22:02:02.494 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=started 2026-02-22T22:02:02.494 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_result 2026-02-22T22:02:02.494 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=stopped 2026-02-22T22:02:02.494 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_41 2026-02-22T22:02:02.494 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_instance=cluster1 2026-02-22T22:02:02.494 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ grep -i state 2026-02-22T22:02:02.495 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:02:02.495 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:02:02.495 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance=cluster1 2026-02-22T22:02:02.495 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local loop_instance 2026-02-22T22:02:02.495 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:02:02.495 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sleep 0 2026-02-22T22:02:02.498 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:02:02.498 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ seq 0 3 2026-02-22T22:02:02.499 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:02:02.499 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_41 2026-02-22T22:02:02.499 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:0 2026-02-22T22:02:02.499 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:02:02.499 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T22:02:02.499 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:02:02.499 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:0 2026-02-22T22:02:02.499 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T22:02:02.499 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T22:02:02.499 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:02:02.499 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=0 2026-02-22T22:02:02.500 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:02:02.500 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T22:02:02.500 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:02:02.500 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=0 2026-02-22T22:02:02.500 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T22:02:02.500 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:02:02.500 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:02:02.500 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T22:02:02.500 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T22:02:02.500 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:02:02.500 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:0 2026-02-22T22:02:02.500 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:02:02.500 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T22:02:02.501 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:02:02.501 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T22:02:02.501 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:02:02.501 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:02:02.501 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T22:02:02.501 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=0 2026-02-22T22:02:02.501 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=0 2026-02-22T22:02:02.501 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:02:02.509 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:02:02.509 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:02:02.510 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:02:02.510 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_41 2026-02-22T22:02:02.557 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ return 0 2026-02-22T22:02:02.558 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:02:02.558 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:02:02.558 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep -i Replaying 2026-02-22T22:02:02.559 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: "state": "Replaying" 2026-02-22T22:02:02.559 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_state=started 2026-02-22T22:02:02.559 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test started = started 2026-02-22T22:02:02.559 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:02:02.559 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_41 2026-02-22T22:02:02.559 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T22:02:02.560 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:02:02.560 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:02:02.560 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_41 2026-02-22T22:02:02.560 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = journal ']' 2026-02-22T22:02:02.560 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T22:02:02.560 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ mirror_image_snapshot cluster2 mirror image_41 2026-02-22T22:02:02.560 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:02:02.560 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:02:02.560 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_41 2026-02-22T22:02:02.560 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_41 2026-02-22T22:02:03.495 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Snapshot ID: 166 2026-02-22T22:02:03.501 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_41 2026-02-22T22:02:03.501 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T22:02:03.501 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:02:03.501 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:02:03.501 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_41 2026-02-22T22:02:03.501 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster2-mirror-image_41.status 2026-02-22T22:02:03.501 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster2-mirror-image_41.status 2026-02-22T22:02:03.501 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:02:03.502 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_log=/tmp/tmp.j44LZPowal/cluster2-mirror-image_41.status 2026-02-22T22:02:03.503 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster1-mirror-image_41.status 2026-02-22T22:02:03.503 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster1-mirror-image_41.status 2026-02-22T22:02:03.503 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:02:03.505 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_status_log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_41.status 2026-02-22T22:02:03.505 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_41 /tmp/tmp.j44LZPowal/cluster2-mirror-image_41.status 2026-02-22T22:02:03.505 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:02:03.505 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:02:03.505 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_41 2026-02-22T22:02:03.505 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster2-mirror-image_41.status 2026-02-22T22:02:03.505 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_41 --format xml 2026-02-22T22:02:03.505 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:02:03.533 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T22:02:03.535 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snapshot_id=166 2026-02-22T22:02:03.535 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T22:02:03.535 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:02:03.535 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.2 2026-02-22T22:02:03.736 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_41 /tmp/tmp.j44LZPowal/cluster1-mirror-image_41.status 2026-02-22T22:02:03.736 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:02:03.736 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:02:03.736 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_41 2026-02-22T22:02:03.736 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_41.status 2026-02-22T22:02:03.737 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_41 --format xml 2026-02-22T22:02:03.737 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:02:03.767 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T22:02:03.769 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id=75 2026-02-22T22:02:03.769 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 166 = 75 2026-02-22T22:02:03.769 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:02:03.769 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.4 2026-02-22T22:02:04.170 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_41 /tmp/tmp.j44LZPowal/cluster1-mirror-image_41.status 2026-02-22T22:02:04.171 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:02:04.171 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:02:04.171 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_41 2026-02-22T22:02:04.171 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_41.status 2026-02-22T22:02:04.171 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_41 --format xml 2026-02-22T22:02:04.171 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:02:04.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T22:02:04.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id=75 2026-02-22T22:02:04.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 166 = 75 2026-02-22T22:02:04.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:02:04.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.8 2026-02-22T22:02:05.002 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_41 /tmp/tmp.j44LZPowal/cluster1-mirror-image_41.status 2026-02-22T22:02:05.003 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:02:05.003 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:02:05.003 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_41 2026-02-22T22:02:05.003 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_41.status 2026-02-22T22:02:05.003 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_41 --format xml 2026-02-22T22:02:05.003 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:02:05.032 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T22:02:05.032 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T22:02:05.033 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T22:02:05.033 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T22:02:05.033 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T22:02:05.034 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T22:02:05.034 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 166 = '' 2026-02-22T22:02:05.034 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:02:05.034 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1.6 2026-02-22T22:02:06.635 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_41 /tmp/tmp.j44LZPowal/cluster1-mirror-image_41.status 2026-02-22T22:02:06.635 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:02:06.636 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:02:06.636 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_41 2026-02-22T22:02:06.636 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_41.status 2026-02-22T22:02:06.636 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_41 --format xml 2026-02-22T22:02:06.636 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:02:06.665 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T22:02:06.666 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id=166 2026-02-22T22:02:06.667 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 166 = 166 2026-02-22T22:02:06.667 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:02:06.667 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snap_present cluster1 mirror image_41 snap 2026-02-22T22:02:06.667 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:02:06.667 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:02:06.667 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_41 2026-02-22T22:02:06.667 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T22:02:06.667 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:02:06.667 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T22:02:06.667 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T22:02:07.668 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 info mirror/image_41@snap 2026-02-22T22:02:07.700 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_41': 2026-02-22T22:02:07.700 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 128 MiB in 32 objects 2026-02-22T22:02:07.700 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T22:02:07.700 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 2 2026-02-22T22:02:07.701 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 119a43e0e16d 2026-02-22T22:02:07.701 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.119a43e0e16d 2026-02-22T22:02:07.701 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T22:02:07.701 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock, non-primary 2026-02-22T22:02:07.701 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T22:02:07.701 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T22:02:07.701 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:54:40 2026 2026-02-22T22:02:07.701 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:54:40 2026 2026-02-22T22:02:07.701 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:54:40 2026 2026-02-22T22:02:07.702 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: protected: False 2026-02-22T22:02:07.702 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T22:02:07.702 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T22:02:07.702 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: 69209f84-531b-4558-bff7-497f660589ca 2026-02-22T22:02:07.702 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: false 2026-02-22T22:02:07.703 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:02:07.703 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ compare_image_snaps mirror image_41 snap 2026-02-22T22:02:07.703 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:02:07.703 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_41 2026-02-22T22:02:07.704 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T22:02:07.704 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local ret=0 2026-02-22T22:02:07.704 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local rmt_export=/tmp/tmp.j44LZPowal/cluster2-mirror-image_41.export 2026-02-22T22:02:07.704 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local loc_export=/tmp/tmp.j44LZPowal/cluster1-mirror-image_41.export 2026-02-22T22:02:07.704 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-image_41.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_41.export 2026-02-22T22:02:07.704 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror export image_41@snap /tmp/tmp.j44LZPowal/cluster2-mirror-image_41.export 2026-02-22T22:02:07.805 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:02:07.812 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 -p mirror export image_41@snap /tmp/tmp.j44LZPowal/cluster1-mirror-image_41.export 2026-02-22T22:02:07.930 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:02:07.937 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ cmp /tmp/tmp.j44LZPowal/cluster2-mirror-image_41.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_41.export 2026-02-22T22:02:07.985 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-image_41.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_41.export 2026-02-22T22:02:08.019 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:02:08.019 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:02:08.019 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_42 2026-02-22T22:02:08.019 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_snap cluster2 mirror image_42 snap 2026-02-22T22:02:08.019 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:02:08.019 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:02:08.019 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_42 2026-02-22T22:02:08.019 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T22:02:08.019 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror snap create image_42@snap --debug-rbd=20 --debug-journaler=20 2026-02-22T22:02:09.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_started cluster1 mirror image_42 2026-02-22T22:02:09.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:02:09.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:02:09.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_42 2026-02-22T22:02:09.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_state cluster1 mirror image_42 started 2026-02-22T22:02:09.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:02:09.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:02:09.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_42 2026-02-22T22:02:09.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=started 2026-02-22T22:02:09.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:02:09.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:02:09.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T22:02:10.203 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_replay_state cluster1 mirror image_42 started 2026-02-22T22:02:10.203 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:02:10.203 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:02:10.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_42 2026-02-22T22:02:10.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=started 2026-02-22T22:02:10.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_result 2026-02-22T22:02:10.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=stopped 2026-02-22T22:02:10.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_42 2026-02-22T22:02:10.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_instance=cluster1 2026-02-22T22:02:10.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:02:10.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:02:10.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance=cluster1 2026-02-22T22:02:10.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local loop_instance 2026-02-22T22:02:10.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ grep -i state 2026-02-22T22:02:10.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:02:10.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sleep 0 2026-02-22T22:02:10.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:02:10.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ seq 0 3 2026-02-22T22:02:10.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:02:10.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_42 2026-02-22T22:02:10.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:0 2026-02-22T22:02:10.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:02:10.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T22:02:10.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:02:10.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:0 2026-02-22T22:02:10.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T22:02:10.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T22:02:10.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:02:10.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=0 2026-02-22T22:02:10.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:02:10.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T22:02:10.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:02:10.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=0 2026-02-22T22:02:10.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T22:02:10.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:02:10.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:02:10.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T22:02:10.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T22:02:10.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:02:10.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:0 2026-02-22T22:02:10.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:02:10.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T22:02:10.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:02:10.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T22:02:10.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:02:10.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:02:10.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T22:02:10.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=0 2026-02-22T22:02:10.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=0 2026-02-22T22:02:10.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:02:10.220 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:02:10.220 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:02:10.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:02:10.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_42 2026-02-22T22:02:10.267 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_42'] 2026-02-22T22:02:10.267 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:02:10.267 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T22:02:10.270 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:02:10.270 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_42 2026-02-22T22:02:10.270 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:1 2026-02-22T22:02:10.270 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:02:10.270 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T22:02:10.270 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:02:10.270 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:1 2026-02-22T22:02:10.270 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T22:02:10.270 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T22:02:10.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:02:10.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=1 2026-02-22T22:02:10.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:02:10.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T22:02:10.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:02:10.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=1 2026-02-22T22:02:10.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T22:02:10.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:02:10.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:1 2026-02-22T22:02:10.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T22:02:10.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T22:02:10.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:02:10.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:1 2026-02-22T22:02:10.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:02:10.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T22:02:10.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:02:10.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T22:02:10.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T22:02:10.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:02:10.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T22:02:10.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=1 2026-02-22T22:02:10.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=1 2026-02-22T22:02:10.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:02:10.281 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:02:10.281 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:02:10.281 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:02:10.281 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_42 2026-02-22T22:02:10.330 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ return 0 2026-02-22T22:02:10.331 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:02:10.331 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:02:10.331 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep -i Replaying 2026-02-22T22:02:10.332 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: "state": "Replaying" 2026-02-22T22:02:10.333 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_state=started 2026-02-22T22:02:10.333 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test started = started 2026-02-22T22:02:10.333 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:02:10.333 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_42 2026-02-22T22:02:10.333 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T22:02:10.333 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:02:10.333 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:02:10.333 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_42 2026-02-22T22:02:10.333 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = journal ']' 2026-02-22T22:02:10.333 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T22:02:10.333 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ mirror_image_snapshot cluster2 mirror image_42 2026-02-22T22:02:10.333 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:02:10.333 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:02:10.333 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_42 2026-02-22T22:02:10.333 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_42 2026-02-22T22:02:11.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Snapshot ID: 168 2026-02-22T22:02:11.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_42 2026-02-22T22:02:11.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T22:02:11.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:02:11.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:02:11.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_42 2026-02-22T22:02:11.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster2-mirror-image_42.status 2026-02-22T22:02:11.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster2-mirror-image_42.status 2026-02-22T22:02:11.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:02:11.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_log=/tmp/tmp.j44LZPowal/cluster2-mirror-image_42.status 2026-02-22T22:02:11.213 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster1-mirror-image_42.status 2026-02-22T22:02:11.213 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster1-mirror-image_42.status 2026-02-22T22:02:11.213 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:02:11.215 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_status_log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_42.status 2026-02-22T22:02:11.215 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_42 /tmp/tmp.j44LZPowal/cluster2-mirror-image_42.status 2026-02-22T22:02:11.215 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:02:11.215 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:02:11.215 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_42 2026-02-22T22:02:11.215 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster2-mirror-image_42.status 2026-02-22T22:02:11.215 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_42 --format xml 2026-02-22T22:02:11.215 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:02:11.242 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T22:02:11.244 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snapshot_id=168 2026-02-22T22:02:11.244 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T22:02:11.244 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:02:11.244 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.2 2026-02-22T22:02:11.446 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_42 /tmp/tmp.j44LZPowal/cluster1-mirror-image_42.status 2026-02-22T22:02:11.446 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:02:11.446 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:02:11.446 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_42 2026-02-22T22:02:11.446 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_42.status 2026-02-22T22:02:11.446 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_42 --format xml 2026-02-22T22:02:11.447 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:02:11.475 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T22:02:11.476 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T22:02:11.477 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T22:02:11.477 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T22:02:11.477 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T22:02:11.478 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T22:02:11.478 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 168 = '' 2026-02-22T22:02:11.478 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:02:11.478 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.4 2026-02-22T22:02:11.879 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_42 /tmp/tmp.j44LZPowal/cluster1-mirror-image_42.status 2026-02-22T22:02:11.879 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:02:11.879 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:02:11.879 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_42 2026-02-22T22:02:11.879 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_42.status 2026-02-22T22:02:11.879 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_42 --format xml 2026-02-22T22:02:11.879 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:02:11.909 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T22:02:11.910 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T22:02:11.910 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T22:02:11.910 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T22:02:11.911 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T22:02:11.911 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T22:02:11.912 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 168 = '' 2026-02-22T22:02:11.912 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:02:11.912 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.8 2026-02-22T22:02:12.712 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_42 /tmp/tmp.j44LZPowal/cluster1-mirror-image_42.status 2026-02-22T22:02:12.713 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:02:12.713 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:02:12.713 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_42 2026-02-22T22:02:12.713 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_42.status 2026-02-22T22:02:12.713 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_42 --format xml 2026-02-22T22:02:12.713 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:02:12.744 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T22:02:12.746 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id=168 2026-02-22T22:02:12.746 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 168 = 168 2026-02-22T22:02:12.746 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:02:12.746 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snap_present cluster1 mirror image_42 snap 2026-02-22T22:02:12.746 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:02:12.746 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:02:12.747 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_42 2026-02-22T22:02:12.747 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T22:02:12.747 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:02:12.747 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T22:02:12.747 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T22:02:13.747 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 info mirror/image_42@snap 2026-02-22T22:02:13.774 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_42': 2026-02-22T22:02:13.774 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 128 MiB in 32 objects 2026-02-22T22:02:13.774 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T22:02:13.774 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 2 2026-02-22T22:02:13.774 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 12277f95b71f 2026-02-22T22:02:13.774 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.12277f95b71f 2026-02-22T22:02:13.774 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T22:02:13.774 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock, non-primary 2026-02-22T22:02:13.774 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T22:02:13.775 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T22:02:13.775 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:54:47 2026 2026-02-22T22:02:13.775 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:54:47 2026 2026-02-22T22:02:13.775 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:54:47 2026 2026-02-22T22:02:13.775 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: protected: False 2026-02-22T22:02:13.775 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T22:02:13.775 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T22:02:13.775 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: 75d1530f-c13b-4f15-aac1-c10cd70d3738 2026-02-22T22:02:13.775 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: false 2026-02-22T22:02:13.777 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:02:13.777 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ compare_image_snaps mirror image_42 snap 2026-02-22T22:02:13.777 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:02:13.777 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_42 2026-02-22T22:02:13.777 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T22:02:13.777 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local ret=0 2026-02-22T22:02:13.777 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local rmt_export=/tmp/tmp.j44LZPowal/cluster2-mirror-image_42.export 2026-02-22T22:02:13.777 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local loc_export=/tmp/tmp.j44LZPowal/cluster1-mirror-image_42.export 2026-02-22T22:02:13.778 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-image_42.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_42.export 2026-02-22T22:02:13.778 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror export image_42@snap /tmp/tmp.j44LZPowal/cluster2-mirror-image_42.export 2026-02-22T22:02:13.886 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:02:13.893 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 -p mirror export image_42@snap /tmp/tmp.j44LZPowal/cluster1-mirror-image_42.export 2026-02-22T22:02:14.012 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:02:14.019 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ cmp /tmp/tmp.j44LZPowal/cluster2-mirror-image_42.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_42.export 2026-02-22T22:02:14.070 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-image_42.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_42.export 2026-02-22T22:02:14.103 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:02:14.103 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:02:14.103 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_43 2026-02-22T22:02:14.103 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_snap cluster2 mirror image_43 snap 2026-02-22T22:02:14.103 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:02:14.104 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:02:14.104 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_43 2026-02-22T22:02:14.104 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T22:02:14.104 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror snap create image_43@snap --debug-rbd=20 --debug-journaler=20 2026-02-22T22:02:15.209 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_started cluster1 mirror image_43 2026-02-22T22:02:15.209 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:02:15.209 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:02:15.209 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_43 2026-02-22T22:02:15.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_state cluster1 mirror image_43 started 2026-02-22T22:02:15.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:02:15.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:02:15.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_43 2026-02-22T22:02:15.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=started 2026-02-22T22:02:15.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:02:15.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:02:15.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T22:02:16.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_replay_state cluster1 mirror image_43 started 2026-02-22T22:02:16.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:02:16.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:02:16.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_43 2026-02-22T22:02:16.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=started 2026-02-22T22:02:16.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_result 2026-02-22T22:02:16.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=stopped 2026-02-22T22:02:16.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_43 2026-02-22T22:02:16.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_instance=cluster1 2026-02-22T22:02:16.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:02:16.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:02:16.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance=cluster1 2026-02-22T22:02:16.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local loop_instance 2026-02-22T22:02:16.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:02:16.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sleep 0 2026-02-22T22:02:16.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ grep -i state 2026-02-22T22:02:16.215 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:02:16.215 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ seq 0 3 2026-02-22T22:02:16.215 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:02:16.215 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_43 2026-02-22T22:02:16.215 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:0 2026-02-22T22:02:16.215 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:02:16.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T22:02:16.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:02:16.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:0 2026-02-22T22:02:16.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T22:02:16.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T22:02:16.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:02:16.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=0 2026-02-22T22:02:16.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:02:16.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T22:02:16.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:02:16.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=0 2026-02-22T22:02:16.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T22:02:16.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:02:16.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:02:16.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T22:02:16.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T22:02:16.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:02:16.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:0 2026-02-22T22:02:16.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:02:16.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T22:02:16.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:02:16.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T22:02:16.218 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:02:16.218 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:02:16.218 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T22:02:16.218 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=0 2026-02-22T22:02:16.218 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=0 2026-02-22T22:02:16.218 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:02:16.225 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:02:16.226 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:02:16.226 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:02:16.226 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_43 2026-02-22T22:02:16.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_43'] 2026-02-22T22:02:16.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:02:16.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T22:02:16.275 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:02:16.275 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_43 2026-02-22T22:02:16.275 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:1 2026-02-22T22:02:16.275 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:02:16.275 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T22:02:16.275 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:02:16.275 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:1 2026-02-22T22:02:16.276 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T22:02:16.276 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T22:02:16.276 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:02:16.276 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=1 2026-02-22T22:02:16.276 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:02:16.276 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T22:02:16.276 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:02:16.276 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=1 2026-02-22T22:02:16.276 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T22:02:16.276 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:02:16.276 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:1 2026-02-22T22:02:16.276 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T22:02:16.277 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T22:02:16.277 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:02:16.277 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:1 2026-02-22T22:02:16.277 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:02:16.277 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T22:02:16.277 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:02:16.277 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T22:02:16.277 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T22:02:16.277 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:02:16.277 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T22:02:16.277 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=1 2026-02-22T22:02:16.277 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=1 2026-02-22T22:02:16.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:02:16.285 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:02:16.285 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:02:16.285 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:02:16.285 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_43 2026-02-22T22:02:16.331 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_43'] 2026-02-22T22:02:16.331 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:02:16.331 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T22:02:16.334 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:02:16.334 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_43 2026-02-22T22:02:16.334 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:2 2026-02-22T22:02:16.334 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:02:16.334 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T22:02:16.334 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:02:16.334 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:2 2026-02-22T22:02:16.334 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T22:02:16.334 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T22:02:16.335 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:02:16.335 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=2 2026-02-22T22:02:16.335 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:02:16.335 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T22:02:16.335 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:02:16.335 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=2 2026-02-22T22:02:16.335 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T22:02:16.335 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:02:16.335 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:2 2026-02-22T22:02:16.335 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T22:02:16.335 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T22:02:16.335 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:02:16.336 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:2 2026-02-22T22:02:16.336 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:02:16.336 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T22:02:16.336 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:02:16.336 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T22:02:16.336 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T22:02:16.336 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:02:16.336 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T22:02:16.336 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=2 2026-02-22T22:02:16.336 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=2 2026-02-22T22:02:16.336 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:02:16.344 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:02:16.345 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:02:16.345 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:02:16.345 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_43 2026-02-22T22:02:16.396 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ return 0 2026-02-22T22:02:16.397 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:02:16.397 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:02:16.398 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep -i Replaying 2026-02-22T22:02:16.398 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: "state": "Replaying" 2026-02-22T22:02:16.400 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_state=started 2026-02-22T22:02:16.401 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test started = started 2026-02-22T22:02:16.401 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:02:16.401 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_43 2026-02-22T22:02:16.401 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T22:02:16.402 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:02:16.402 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:02:16.402 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_43 2026-02-22T22:02:16.402 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = journal ']' 2026-02-22T22:02:16.402 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T22:02:16.402 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ mirror_image_snapshot cluster2 mirror image_43 2026-02-22T22:02:16.403 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:02:16.403 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:02:16.403 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_43 2026-02-22T22:02:16.403 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_43 2026-02-22T22:02:17.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Snapshot ID: 170 2026-02-22T22:02:17.215 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_43 2026-02-22T22:02:17.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T22:02:17.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:02:17.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:02:17.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_43 2026-02-22T22:02:17.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster2-mirror-image_43.status 2026-02-22T22:02:17.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster2-mirror-image_43.status 2026-02-22T22:02:17.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:02:17.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_log=/tmp/tmp.j44LZPowal/cluster2-mirror-image_43.status 2026-02-22T22:02:17.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster1-mirror-image_43.status 2026-02-22T22:02:17.218 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster1-mirror-image_43.status 2026-02-22T22:02:17.218 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:02:17.219 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_status_log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_43.status 2026-02-22T22:02:17.219 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_43 /tmp/tmp.j44LZPowal/cluster2-mirror-image_43.status 2026-02-22T22:02:17.219 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:02:17.219 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:02:17.219 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_43 2026-02-22T22:02:17.220 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster2-mirror-image_43.status 2026-02-22T22:02:17.220 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_43 --format xml 2026-02-22T22:02:17.220 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:02:17.247 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T22:02:17.248 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snapshot_id=170 2026-02-22T22:02:17.248 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T22:02:17.248 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:02:17.249 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.2 2026-02-22T22:02:17.450 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_43 /tmp/tmp.j44LZPowal/cluster1-mirror-image_43.status 2026-02-22T22:02:17.450 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:02:17.450 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:02:17.450 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_43 2026-02-22T22:02:17.450 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_43.status 2026-02-22T22:02:17.450 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_43 --format xml 2026-02-22T22:02:17.450 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:02:17.478 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T22:02:17.478 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T22:02:17.479 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T22:02:17.479 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T22:02:17.479 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T22:02:17.480 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T22:02:17.480 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 170 = '' 2026-02-22T22:02:17.480 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:02:17.480 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.4 2026-02-22T22:02:17.880 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_43 /tmp/tmp.j44LZPowal/cluster1-mirror-image_43.status 2026-02-22T22:02:17.881 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:02:17.881 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:02:17.881 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_43 2026-02-22T22:02:17.881 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_43.status 2026-02-22T22:02:17.881 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_43 --format xml 2026-02-22T22:02:17.881 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:02:17.910 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T22:02:17.910 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T22:02:17.911 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T22:02:17.911 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T22:02:17.911 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T22:02:17.912 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T22:02:17.912 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 170 = '' 2026-02-22T22:02:17.912 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:02:17.912 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.8 2026-02-22T22:02:18.713 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_43 /tmp/tmp.j44LZPowal/cluster1-mirror-image_43.status 2026-02-22T22:02:18.713 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:02:18.713 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:02:18.713 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_43 2026-02-22T22:02:18.714 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_43.status 2026-02-22T22:02:18.714 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_43 --format xml 2026-02-22T22:02:18.714 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:02:18.744 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T22:02:18.746 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id=170 2026-02-22T22:02:18.746 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 170 = 170 2026-02-22T22:02:18.746 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:02:18.746 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snap_present cluster1 mirror image_43 snap 2026-02-22T22:02:18.746 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:02:18.747 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:02:18.747 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_43 2026-02-22T22:02:18.747 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T22:02:18.747 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:02:18.747 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T22:02:18.747 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T22:02:19.747 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 info mirror/image_43@snap 2026-02-22T22:02:19.773 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_43': 2026-02-22T22:02:19.773 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 128 MiB in 32 objects 2026-02-22T22:02:19.773 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T22:02:19.773 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 2 2026-02-22T22:02:19.773 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 126fe6d46e9d 2026-02-22T22:02:19.773 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.126fe6d46e9d 2026-02-22T22:02:19.774 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T22:02:19.774 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock, non-primary 2026-02-22T22:02:19.774 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T22:02:19.774 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T22:02:19.774 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:54:56 2026 2026-02-22T22:02:19.774 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:54:56 2026 2026-02-22T22:02:19.774 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:54:56 2026 2026-02-22T22:02:19.774 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: protected: False 2026-02-22T22:02:19.774 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T22:02:19.774 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T22:02:19.774 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: af18f3fc-e37e-4800-a4fb-c06589d02d67 2026-02-22T22:02:19.775 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: false 2026-02-22T22:02:19.776 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:02:19.776 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ compare_image_snaps mirror image_43 snap 2026-02-22T22:02:19.776 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:02:19.776 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_43 2026-02-22T22:02:19.776 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T22:02:19.776 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local ret=0 2026-02-22T22:02:19.777 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local rmt_export=/tmp/tmp.j44LZPowal/cluster2-mirror-image_43.export 2026-02-22T22:02:19.777 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local loc_export=/tmp/tmp.j44LZPowal/cluster1-mirror-image_43.export 2026-02-22T22:02:19.777 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-image_43.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_43.export 2026-02-22T22:02:19.777 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror export image_43@snap /tmp/tmp.j44LZPowal/cluster2-mirror-image_43.export 2026-02-22T22:02:19.878 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:02:19.885 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 -p mirror export image_43@snap /tmp/tmp.j44LZPowal/cluster1-mirror-image_43.export 2026-02-22T22:02:19.996 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:02:20.004 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ cmp /tmp/tmp.j44LZPowal/cluster2-mirror-image_43.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_43.export 2026-02-22T22:02:20.051 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-image_43.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_43.export 2026-02-22T22:02:20.086 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:02:20.086 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:02:20.086 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_44 2026-02-22T22:02:20.087 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_snap cluster2 mirror image_44 snap 2026-02-22T22:02:20.087 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:02:20.087 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:02:20.087 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_44 2026-02-22T22:02:20.087 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T22:02:20.087 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror snap create image_44@snap --debug-rbd=20 --debug-journaler=20 2026-02-22T22:02:21.266 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_started cluster1 mirror image_44 2026-02-22T22:02:21.267 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:02:21.267 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:02:21.267 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_44 2026-02-22T22:02:21.267 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_state cluster1 mirror image_44 started 2026-02-22T22:02:21.267 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:02:21.267 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:02:21.267 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_44 2026-02-22T22:02:21.268 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=started 2026-02-22T22:02:21.268 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:02:21.268 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:02:21.268 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T22:02:22.267 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_replay_state cluster1 mirror image_44 started 2026-02-22T22:02:22.267 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:02:22.267 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:02:22.267 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_44 2026-02-22T22:02:22.267 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=started 2026-02-22T22:02:22.267 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_result 2026-02-22T22:02:22.267 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=stopped 2026-02-22T22:02:22.267 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_44 2026-02-22T22:02:22.268 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ grep -i state 2026-02-22T22:02:22.268 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_instance=cluster1 2026-02-22T22:02:22.268 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:02:22.268 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:02:22.268 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance=cluster1 2026-02-22T22:02:22.268 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local loop_instance 2026-02-22T22:02:22.268 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:02:22.268 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sleep 0 2026-02-22T22:02:22.268 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:02:22.268 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ seq 0 3 2026-02-22T22:02:22.269 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:02:22.269 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_44 2026-02-22T22:02:22.269 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:0 2026-02-22T22:02:22.269 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:02:22.269 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T22:02:22.270 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:02:22.270 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:0 2026-02-22T22:02:22.270 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T22:02:22.270 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T22:02:22.270 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:02:22.270 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=0 2026-02-22T22:02:22.270 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:02:22.270 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T22:02:22.270 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:02:22.270 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=0 2026-02-22T22:02:22.270 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T22:02:22.270 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:02:22.270 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:02:22.270 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T22:02:22.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T22:02:22.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:02:22.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:0 2026-02-22T22:02:22.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:02:22.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T22:02:22.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:02:22.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T22:02:22.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:02:22.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:02:22.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T22:02:22.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=0 2026-02-22T22:02:22.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=0 2026-02-22T22:02:22.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:02:22.282 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:02:22.282 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:02:22.282 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:02:22.282 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_44 2026-02-22T22:02:22.329 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_44'] 2026-02-22T22:02:22.329 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:02:22.329 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T22:02:22.331 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:02:22.331 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_44 2026-02-22T22:02:22.331 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:1 2026-02-22T22:02:22.331 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:02:22.331 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T22:02:22.331 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:02:22.331 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:1 2026-02-22T22:02:22.331 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T22:02:22.331 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T22:02:22.331 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:02:22.331 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=1 2026-02-22T22:02:22.331 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:02:22.331 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T22:02:22.331 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:02:22.331 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=1 2026-02-22T22:02:22.331 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T22:02:22.332 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:02:22.332 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:1 2026-02-22T22:02:22.332 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T22:02:22.332 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T22:02:22.332 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:02:22.332 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:1 2026-02-22T22:02:22.332 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:02:22.332 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T22:02:22.332 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:02:22.332 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T22:02:22.332 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T22:02:22.332 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:02:22.332 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T22:02:22.332 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=1 2026-02-22T22:02:22.332 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=1 2026-02-22T22:02:22.332 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:02:22.343 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:02:22.343 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:02:22.343 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:02:22.343 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_44 2026-02-22T22:02:22.392 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ return 0 2026-02-22T22:02:22.393 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:02:22.393 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:02:22.393 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep -i Replaying 2026-02-22T22:02:22.394 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: "state": "Replaying" 2026-02-22T22:02:22.394 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_state=started 2026-02-22T22:02:22.394 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test started = started 2026-02-22T22:02:22.394 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:02:22.395 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_44 2026-02-22T22:02:22.395 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T22:02:22.395 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:02:22.395 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:02:22.395 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_44 2026-02-22T22:02:22.395 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = journal ']' 2026-02-22T22:02:22.395 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T22:02:22.395 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ mirror_image_snapshot cluster2 mirror image_44 2026-02-22T22:02:22.395 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:02:22.395 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:02:22.395 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_44 2026-02-22T22:02:22.395 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_44 2026-02-22T22:02:23.269 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Snapshot ID: 172 2026-02-22T22:02:23.276 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_44 2026-02-22T22:02:23.276 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T22:02:23.276 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:02:23.276 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:02:23.276 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_44 2026-02-22T22:02:23.276 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster2-mirror-image_44.status 2026-02-22T22:02:23.276 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster2-mirror-image_44.status 2026-02-22T22:02:23.276 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:02:23.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_log=/tmp/tmp.j44LZPowal/cluster2-mirror-image_44.status 2026-02-22T22:02:23.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster1-mirror-image_44.status 2026-02-22T22:02:23.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster1-mirror-image_44.status 2026-02-22T22:02:23.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:02:23.279 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_status_log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_44.status 2026-02-22T22:02:23.279 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_44 /tmp/tmp.j44LZPowal/cluster2-mirror-image_44.status 2026-02-22T22:02:23.279 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:02:23.280 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:02:23.280 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_44 2026-02-22T22:02:23.280 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster2-mirror-image_44.status 2026-02-22T22:02:23.280 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_44 --format xml 2026-02-22T22:02:23.280 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:02:23.307 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T22:02:23.308 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snapshot_id=172 2026-02-22T22:02:23.309 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T22:02:23.309 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:02:23.309 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.2 2026-02-22T22:02:23.509 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_44 /tmp/tmp.j44LZPowal/cluster1-mirror-image_44.status 2026-02-22T22:02:23.509 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:02:23.509 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:02:23.509 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_44 2026-02-22T22:02:23.509 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_44.status 2026-02-22T22:02:23.509 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_44 --format xml 2026-02-22T22:02:23.509 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:02:23.539 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T22:02:23.539 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T22:02:23.540 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T22:02:23.540 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T22:02:23.540 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T22:02:23.541 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T22:02:23.541 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 172 = '' 2026-02-22T22:02:23.541 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:02:23.541 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.4 2026-02-22T22:02:23.942 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_44 /tmp/tmp.j44LZPowal/cluster1-mirror-image_44.status 2026-02-22T22:02:23.943 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:02:23.943 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:02:23.943 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_44 2026-02-22T22:02:23.943 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_44.status 2026-02-22T22:02:23.943 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_44 --format xml 2026-02-22T22:02:23.943 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:02:23.975 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T22:02:23.976 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T22:02:23.977 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T22:02:23.977 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T22:02:23.977 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T22:02:23.978 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T22:02:23.978 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 172 = '' 2026-02-22T22:02:23.978 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:02:23.978 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.8 2026-02-22T22:02:24.779 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_44 /tmp/tmp.j44LZPowal/cluster1-mirror-image_44.status 2026-02-22T22:02:24.779 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:02:24.779 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:02:24.779 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_44 2026-02-22T22:02:24.780 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_44.status 2026-02-22T22:02:24.780 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_44 --format xml 2026-02-22T22:02:24.780 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:02:24.812 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T22:02:24.813 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id=172 2026-02-22T22:02:24.813 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 172 = 172 2026-02-22T22:02:24.813 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:02:24.813 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snap_present cluster1 mirror image_44 snap 2026-02-22T22:02:24.813 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:02:24.813 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:02:24.813 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_44 2026-02-22T22:02:24.813 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T22:02:24.813 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:02:24.814 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T22:02:24.814 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T22:02:25.814 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 info mirror/image_44@snap 2026-02-22T22:02:25.841 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_44': 2026-02-22T22:02:25.842 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 128 MiB in 32 objects 2026-02-22T22:02:25.842 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T22:02:25.842 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 2 2026-02-22T22:02:25.842 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 126fa73a0af5 2026-02-22T22:02:25.842 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.126fa73a0af5 2026-02-22T22:02:25.842 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T22:02:25.842 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock, non-primary 2026-02-22T22:02:25.842 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T22:02:25.842 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T22:02:25.842 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:55:07 2026 2026-02-22T22:02:25.842 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:55:07 2026 2026-02-22T22:02:25.843 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:55:07 2026 2026-02-22T22:02:25.843 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: protected: False 2026-02-22T22:02:25.843 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T22:02:25.843 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T22:02:25.843 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: 1f09f5dc-1ede-4e63-bd97-5b2825b46268 2026-02-22T22:02:25.843 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: false 2026-02-22T22:02:25.844 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:02:25.844 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ compare_image_snaps mirror image_44 snap 2026-02-22T22:02:25.844 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:02:25.844 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_44 2026-02-22T22:02:25.845 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T22:02:25.845 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local ret=0 2026-02-22T22:02:25.845 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local rmt_export=/tmp/tmp.j44LZPowal/cluster2-mirror-image_44.export 2026-02-22T22:02:25.845 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local loc_export=/tmp/tmp.j44LZPowal/cluster1-mirror-image_44.export 2026-02-22T22:02:25.845 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-image_44.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_44.export 2026-02-22T22:02:25.845 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror export image_44@snap /tmp/tmp.j44LZPowal/cluster2-mirror-image_44.export 2026-02-22T22:02:25.947 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:02:25.953 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 -p mirror export image_44@snap /tmp/tmp.j44LZPowal/cluster1-mirror-image_44.export 2026-02-22T22:02:26.081 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:02:26.088 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ cmp /tmp/tmp.j44LZPowal/cluster2-mirror-image_44.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_44.export 2026-02-22T22:02:26.138 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-image_44.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_44.export 2026-02-22T22:02:26.171 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:02:26.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:02:26.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_45 2026-02-22T22:02:26.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_snap cluster2 mirror image_45 snap 2026-02-22T22:02:26.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:02:26.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:02:26.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_45 2026-02-22T22:02:26.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T22:02:26.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror snap create image_45@snap --debug-rbd=20 --debug-journaler=20 2026-02-22T22:02:27.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_started cluster1 mirror image_45 2026-02-22T22:02:27.273 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:02:27.273 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:02:27.273 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_45 2026-02-22T22:02:27.273 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_state cluster1 mirror image_45 started 2026-02-22T22:02:27.273 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:02:27.273 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:02:27.273 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_45 2026-02-22T22:02:27.273 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=started 2026-02-22T22:02:27.273 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:02:27.273 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:02:27.274 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T22:02:28.274 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_replay_state cluster1 mirror image_45 started 2026-02-22T22:02:28.274 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:02:28.275 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:02:28.275 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_45 2026-02-22T22:02:28.275 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=started 2026-02-22T22:02:28.275 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_result 2026-02-22T22:02:28.275 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=stopped 2026-02-22T22:02:28.275 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_45 2026-02-22T22:02:28.275 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_instance=cluster1 2026-02-22T22:02:28.275 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:02:28.275 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:02:28.276 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance=cluster1 2026-02-22T22:02:28.276 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local loop_instance 2026-02-22T22:02:28.276 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:02:28.276 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sleep 0 2026-02-22T22:02:28.276 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ grep -i state 2026-02-22T22:02:28.279 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:02:28.279 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ seq 0 3 2026-02-22T22:02:28.279 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:02:28.279 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_45 2026-02-22T22:02:28.279 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:0 2026-02-22T22:02:28.279 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:02:28.279 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T22:02:28.280 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:02:28.280 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:0 2026-02-22T22:02:28.280 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T22:02:28.280 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T22:02:28.280 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:02:28.280 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=0 2026-02-22T22:02:28.280 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:02:28.280 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T22:02:28.280 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:02:28.280 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=0 2026-02-22T22:02:28.280 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T22:02:28.281 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:02:28.281 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:02:28.281 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T22:02:28.281 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T22:02:28.281 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:02:28.281 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:0 2026-02-22T22:02:28.281 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:02:28.281 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T22:02:28.281 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:02:28.281 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T22:02:28.282 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:02:28.282 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:02:28.282 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T22:02:28.282 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=0 2026-02-22T22:02:28.282 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=0 2026-02-22T22:02:28.282 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:02:28.289 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:02:28.289 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:02:28.289 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:02:28.289 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_45 2026-02-22T22:02:28.338 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ return 0 2026-02-22T22:02:28.339 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:02:28.339 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:02:28.339 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep -i Replaying 2026-02-22T22:02:28.340 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: "state": "Replaying" 2026-02-22T22:02:28.341 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_state=started 2026-02-22T22:02:28.341 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test started = started 2026-02-22T22:02:28.341 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:02:28.341 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_45 2026-02-22T22:02:28.341 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T22:02:28.341 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:02:28.341 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:02:28.342 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_45 2026-02-22T22:02:28.342 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = journal ']' 2026-02-22T22:02:28.342 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T22:02:28.342 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ mirror_image_snapshot cluster2 mirror image_45 2026-02-22T22:02:28.342 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:02:28.342 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:02:28.342 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_45 2026-02-22T22:02:28.342 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_45 2026-02-22T22:02:29.276 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Snapshot ID: 174 2026-02-22T22:02:29.282 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_45 2026-02-22T22:02:29.282 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T22:02:29.282 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:02:29.282 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:02:29.282 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_45 2026-02-22T22:02:29.282 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster2-mirror-image_45.status 2026-02-22T22:02:29.282 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster2-mirror-image_45.status 2026-02-22T22:02:29.282 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:02:29.283 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_log=/tmp/tmp.j44LZPowal/cluster2-mirror-image_45.status 2026-02-22T22:02:29.283 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster1-mirror-image_45.status 2026-02-22T22:02:29.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster1-mirror-image_45.status 2026-02-22T22:02:29.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:02:29.285 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_status_log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_45.status 2026-02-22T22:02:29.285 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_45 /tmp/tmp.j44LZPowal/cluster2-mirror-image_45.status 2026-02-22T22:02:29.285 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:02:29.285 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:02:29.285 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_45 2026-02-22T22:02:29.286 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster2-mirror-image_45.status 2026-02-22T22:02:29.286 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_45 --format xml 2026-02-22T22:02:29.286 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:02:29.313 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T22:02:29.315 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snapshot_id=174 2026-02-22T22:02:29.315 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T22:02:29.315 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:02:29.315 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.2 2026-02-22T22:02:29.516 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_45 /tmp/tmp.j44LZPowal/cluster1-mirror-image_45.status 2026-02-22T22:02:29.516 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:02:29.516 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:02:29.516 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_45 2026-02-22T22:02:29.517 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_45.status 2026-02-22T22:02:29.517 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_45 --format xml 2026-02-22T22:02:29.517 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:02:29.544 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T22:02:29.545 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T22:02:29.546 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T22:02:29.546 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T22:02:29.546 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T22:02:29.547 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T22:02:29.547 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 174 = '' 2026-02-22T22:02:29.547 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:02:29.547 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.4 2026-02-22T22:02:29.947 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_45 /tmp/tmp.j44LZPowal/cluster1-mirror-image_45.status 2026-02-22T22:02:29.947 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:02:29.947 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:02:29.948 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_45 2026-02-22T22:02:29.948 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_45.status 2026-02-22T22:02:29.948 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_45 --format xml 2026-02-22T22:02:29.948 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:02:29.977 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T22:02:29.977 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T22:02:29.978 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T22:02:29.978 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T22:02:29.978 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T22:02:29.979 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T22:02:29.979 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 174 = '' 2026-02-22T22:02:29.979 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:02:29.979 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.8 2026-02-22T22:02:30.780 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_45 /tmp/tmp.j44LZPowal/cluster1-mirror-image_45.status 2026-02-22T22:02:30.780 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:02:30.780 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:02:30.781 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_45 2026-02-22T22:02:30.781 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_45.status 2026-02-22T22:02:30.781 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_45 --format xml 2026-02-22T22:02:30.781 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:02:30.811 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T22:02:30.813 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id=174 2026-02-22T22:02:30.814 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 174 = 174 2026-02-22T22:02:30.814 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:02:30.814 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snap_present cluster1 mirror image_45 snap 2026-02-22T22:02:30.814 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:02:30.814 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:02:30.814 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_45 2026-02-22T22:02:30.814 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T22:02:30.814 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:02:30.814 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T22:02:30.814 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T22:02:31.814 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 info mirror/image_45@snap 2026-02-22T22:02:31.841 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_45': 2026-02-22T22:02:31.841 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 128 MiB in 32 objects 2026-02-22T22:02:31.841 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T22:02:31.841 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 2 2026-02-22T22:02:31.841 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 119a947044bc 2026-02-22T22:02:31.841 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.119a947044bc 2026-02-22T22:02:31.841 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T22:02:31.841 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock, non-primary 2026-02-22T22:02:31.841 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T22:02:31.841 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T22:02:31.841 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:55:19 2026 2026-02-22T22:02:31.841 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:55:19 2026 2026-02-22T22:02:31.841 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:55:19 2026 2026-02-22T22:02:31.841 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: protected: False 2026-02-22T22:02:31.841 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T22:02:31.841 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T22:02:31.841 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: 1024109e-7bb5-4ed3-ab78-ad539d98c9fd 2026-02-22T22:02:31.841 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: false 2026-02-22T22:02:31.843 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:02:31.843 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ compare_image_snaps mirror image_45 snap 2026-02-22T22:02:31.843 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:02:31.843 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_45 2026-02-22T22:02:31.843 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T22:02:31.843 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local ret=0 2026-02-22T22:02:31.843 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local rmt_export=/tmp/tmp.j44LZPowal/cluster2-mirror-image_45.export 2026-02-22T22:02:31.843 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local loc_export=/tmp/tmp.j44LZPowal/cluster1-mirror-image_45.export 2026-02-22T22:02:31.843 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-image_45.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_45.export 2026-02-22T22:02:31.844 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror export image_45@snap /tmp/tmp.j44LZPowal/cluster2-mirror-image_45.export 2026-02-22T22:02:31.946 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:02:31.951 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 -p mirror export image_45@snap /tmp/tmp.j44LZPowal/cluster1-mirror-image_45.export 2026-02-22T22:02:32.076 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:02:32.083 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ cmp /tmp/tmp.j44LZPowal/cluster2-mirror-image_45.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_45.export 2026-02-22T22:02:32.132 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-image_45.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_45.export 2026-02-22T22:02:32.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:02:32.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:02:32.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_46 2026-02-22T22:02:32.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_snap cluster2 mirror image_46 snap 2026-02-22T22:02:32.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:02:32.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:02:32.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_46 2026-02-22T22:02:32.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T22:02:32.169 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror snap create image_46@snap --debug-rbd=20 --debug-journaler=20 2026-02-22T22:02:32.560 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.8616945954071034) > weight (0.25) 2026-02-22T22:02:32.560 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.1 2026-02-22T22:02:32.560 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Sent signal 15 2026-02-22T22:02:32.560 INFO:tasks.rbd_mirror_thrash:waiting for 2.729871218741998 secs before reviving daemons 2026-02-22T22:02:32.560 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial026.stderr:2026-02-22T22:02:32.559+0000 7f5e30cb7640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 (PID: 68495) UID: 1000 2026-02-22T22:02:32.582 INFO:tasks.ceph.cluster2.osd.2.trial026.stderr:2026-02-22T22:02:32.581+0000 7f32e29d2640 -1 reset not still connected to 0x5576a496a4e0 2026-02-22T22:02:32.583 INFO:tasks.ceph.cluster2.osd.2.trial026.stderr:2026-02-22T22:02:32.582+0000 7f32e29d2640 -1 reset not still connected to 0x5576a5d18270 2026-02-22T22:02:32.584 INFO:tasks.ceph.cluster2.osd.2.trial026.stderr:2026-02-22T22:02:32.583+0000 7f32e29d2640 -1 reset not still connected to 0x5576ad25ba00 2026-02-22T22:02:33.281 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_started cluster1 mirror image_46 2026-02-22T22:02:33.281 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:02:33.281 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:02:33.281 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_46 2026-02-22T22:02:33.281 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_state cluster1 mirror image_46 started 2026-02-22T22:02:33.281 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:02:33.281 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:02:33.282 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_46 2026-02-22T22:02:33.282 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=started 2026-02-22T22:02:33.282 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:02:33.282 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:02:33.282 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T22:02:34.282 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_replay_state cluster1 mirror image_46 started 2026-02-22T22:02:34.282 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:02:34.283 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:02:34.283 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_46 2026-02-22T22:02:34.283 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=started 2026-02-22T22:02:34.283 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_result 2026-02-22T22:02:34.283 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=stopped 2026-02-22T22:02:34.283 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_46 2026-02-22T22:02:34.283 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_instance=cluster1 2026-02-22T22:02:34.283 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:02:34.283 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:02:34.283 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance=cluster1 2026-02-22T22:02:34.283 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local loop_instance 2026-02-22T22:02:34.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ grep -i state 2026-02-22T22:02:34.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:02:34.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sleep 0 2026-02-22T22:02:34.287 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:02:34.287 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ seq 0 3 2026-02-22T22:02:34.287 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:02:34.287 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_46 2026-02-22T22:02:34.287 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:0 2026-02-22T22:02:34.287 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:02:34.287 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T22:02:34.287 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:02:34.287 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:0 2026-02-22T22:02:34.288 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T22:02:34.288 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T22:02:34.288 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:02:34.288 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=0 2026-02-22T22:02:34.288 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:02:34.288 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T22:02:34.288 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:02:34.288 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=0 2026-02-22T22:02:34.288 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T22:02:34.288 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:02:34.288 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:02:34.289 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T22:02:34.289 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T22:02:34.289 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:02:34.289 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:0 2026-02-22T22:02:34.289 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:02:34.289 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T22:02:34.289 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:02:34.289 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T22:02:34.289 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:02:34.289 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:02:34.289 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T22:02:34.289 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=0 2026-02-22T22:02:34.289 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=0 2026-02-22T22:02:34.290 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:02:34.297 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:02:34.297 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:02:34.297 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:02:34.297 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_46 2026-02-22T22:02:34.341 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_46'] 2026-02-22T22:02:34.342 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:02:34.342 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T22:02:34.343 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:02:34.343 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_46 2026-02-22T22:02:34.343 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:1 2026-02-22T22:02:34.343 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:02:34.343 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T22:02:34.343 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:02:34.343 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:1 2026-02-22T22:02:34.343 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T22:02:34.343 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T22:02:34.343 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:02:34.343 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=1 2026-02-22T22:02:34.343 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:02:34.343 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T22:02:34.343 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:02:34.344 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=1 2026-02-22T22:02:34.344 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T22:02:34.344 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:02:34.344 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:1 2026-02-22T22:02:34.344 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T22:02:34.344 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T22:02:34.344 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:02:34.344 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:1 2026-02-22T22:02:34.344 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:02:34.344 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T22:02:34.344 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:02:34.344 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T22:02:34.344 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T22:02:34.344 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:02:34.344 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T22:02:34.344 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=1 2026-02-22T22:02:34.344 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=1 2026-02-22T22:02:34.344 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:02:34.355 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:02:34.355 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:02:34.355 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:02:34.355 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_46 2026-02-22T22:02:34.392 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-22T22:02:34.393 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:02:34.393 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_46 2026-02-22T22:02:34.393 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:2 2026-02-22T22:02:34.393 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:02:34.393 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T22:02:34.393 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:02:34.393 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:2 2026-02-22T22:02:34.393 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T22:02:34.393 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T22:02:34.393 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:02:34.393 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=2 2026-02-22T22:02:34.393 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:02:34.393 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T22:02:34.393 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:02:34.393 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=2 2026-02-22T22:02:34.394 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T22:02:34.394 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:02:34.394 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:2 2026-02-22T22:02:34.394 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T22:02:34.394 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T22:02:34.394 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:02:34.394 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:2 2026-02-22T22:02:34.394 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:02:34.394 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T22:02:34.394 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:02:34.394 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T22:02:34.394 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T22:02:34.394 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:02:34.394 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T22:02:34.394 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=2 2026-02-22T22:02:34.394 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=2 2026-02-22T22:02:34.394 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:02:34.404 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:02:34.405 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:02:34.405 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:02:34.405 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_46 2026-02-22T22:02:34.452 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_46'] 2026-02-22T22:02:34.452 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:02:34.452 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T22:02:34.454 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:02:34.454 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_46 2026-02-22T22:02:34.454 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:3 2026-02-22T22:02:34.454 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:02:34.454 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T22:02:34.454 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T22:02:34.454 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:3 2026-02-22T22:02:34.454 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T22:02:34.454 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T22:02:34.454 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:02:34.454 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=3 2026-02-22T22:02:34.454 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T22:02:34.454 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T22:02:34.454 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:02:34.454 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=3 2026-02-22T22:02:34.454 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=3 2026-02-22T22:02:34.454 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T22:02:34.455 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:3 2026-02-22T22:02:34.455 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T22:02:34.455 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T22:02:34.455 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T22:02:34.455 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:3 2026-02-22T22:02:34.455 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:02:34.455 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T22:02:34.455 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:02:34.455 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=3 2026-02-22T22:02:34.455 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T22:02:34.455 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:02:34.455 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T22:02:34.455 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=3 2026-02-22T22:02:34.455 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=3 2026-02-22T22:02:34.455 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T22:02:34.465 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:02:34.466 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:02:34.466 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:02:34.466 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_46 2026-02-22T22:02:34.513 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_46'] 2026-02-22T22:02:34.513 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:02:34.513 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T22:02:34.514 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:02:34.514 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sleep 1 2026-02-22T22:02:35.291 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.1 2026-02-22T22:02:35.291 INFO:teuthology.orchestra.run:waiting for 600 2026-02-22T22:02:35.291 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.1 2026-02-22T22:02:35.292 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Restarting daemon 2026-02-22T22:02:35.292 DEBUG:teuthology.orchestra.run.trial026:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 2026-02-22T22:02:35.294 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Started 2026-02-22T22:02:35.295 INFO:tasks.rbd_mirror_thrash:waiting for 102.2187799431019 secs before thrashing 2026-02-22T22:02:35.515 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:02:35.515 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ seq 0 3 2026-02-22T22:02:35.517 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:02:35.517 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_46 2026-02-22T22:02:35.517 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:0 2026-02-22T22:02:35.517 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:02:35.517 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T22:02:35.517 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:02:35.517 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:0 2026-02-22T22:02:35.517 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T22:02:35.517 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T22:02:35.517 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:02:35.517 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=0 2026-02-22T22:02:35.517 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:02:35.517 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T22:02:35.517 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:02:35.517 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=0 2026-02-22T22:02:35.517 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T22:02:35.517 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:02:35.517 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:02:35.517 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T22:02:35.517 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T22:02:35.517 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:02:35.517 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:0 2026-02-22T22:02:35.517 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:02:35.517 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T22:02:35.517 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:02:35.517 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T22:02:35.517 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:02:35.517 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:02:35.517 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T22:02:35.517 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=0 2026-02-22T22:02:35.517 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=0 2026-02-22T22:02:35.517 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:02:35.528 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:02:35.529 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:02:35.529 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:02:35.529 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_46 2026-02-22T22:02:35.574 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_46'] 2026-02-22T22:02:35.574 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:02:35.575 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T22:02:35.579 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:02:35.579 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_46 2026-02-22T22:02:35.579 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:1 2026-02-22T22:02:35.580 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:02:35.580 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T22:02:35.580 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:02:35.580 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:1 2026-02-22T22:02:35.580 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T22:02:35.580 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T22:02:35.580 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:02:35.581 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=1 2026-02-22T22:02:35.581 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:02:35.581 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T22:02:35.581 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:02:35.581 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=1 2026-02-22T22:02:35.581 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T22:02:35.581 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:02:35.581 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:1 2026-02-22T22:02:35.582 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T22:02:35.582 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T22:02:35.582 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:02:35.582 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:1 2026-02-22T22:02:35.582 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:02:35.582 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T22:02:35.582 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:02:35.582 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T22:02:35.583 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T22:02:35.583 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:02:35.583 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T22:02:35.583 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=1 2026-02-22T22:02:35.583 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=1 2026-02-22T22:02:35.583 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:02:35.588 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:02:35.588 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:02:35.588 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:02:35.588 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_46 2026-02-22T22:02:35.627 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_46'] 2026-02-22T22:02:35.628 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:02:35.628 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T22:02:35.630 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:02:35.631 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_46 2026-02-22T22:02:35.631 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:2 2026-02-22T22:02:35.631 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:02:35.631 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T22:02:35.631 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:02:35.631 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:2 2026-02-22T22:02:35.631 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T22:02:35.631 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T22:02:35.631 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:02:35.631 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=2 2026-02-22T22:02:35.632 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:02:35.632 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T22:02:35.632 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:02:35.632 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=2 2026-02-22T22:02:35.632 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T22:02:35.632 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:02:35.632 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:2 2026-02-22T22:02:35.632 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T22:02:35.632 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T22:02:35.632 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:02:35.632 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:2 2026-02-22T22:02:35.632 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:02:35.633 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T22:02:35.633 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:02:35.633 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T22:02:35.633 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T22:02:35.633 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:02:35.633 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T22:02:35.633 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=2 2026-02-22T22:02:35.633 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=2 2026-02-22T22:02:35.633 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:02:35.641 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:02:35.641 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:02:35.641 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:02:35.642 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_46 2026-02-22T22:02:35.686 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_46'] 2026-02-22T22:02:35.687 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:02:35.687 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T22:02:35.689 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:02:35.689 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_46 2026-02-22T22:02:35.689 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:3 2026-02-22T22:02:35.690 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:02:35.690 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T22:02:35.690 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T22:02:35.690 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:3 2026-02-22T22:02:35.690 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T22:02:35.690 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T22:02:35.690 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:02:35.690 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=3 2026-02-22T22:02:35.690 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T22:02:35.690 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T22:02:35.690 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:02:35.691 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=3 2026-02-22T22:02:35.691 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=3 2026-02-22T22:02:35.691 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T22:02:35.691 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:3 2026-02-22T22:02:35.691 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T22:02:35.691 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T22:02:35.691 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T22:02:35.691 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:3 2026-02-22T22:02:35.691 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:02:35.691 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T22:02:35.691 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:02:35.691 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=3 2026-02-22T22:02:35.691 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T22:02:35.692 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:02:35.692 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T22:02:35.692 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=3 2026-02-22T22:02:35.692 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=3 2026-02-22T22:02:35.692 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T22:02:35.700 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:02:35.700 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:02:35.700 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:02:35.700 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_46 2026-02-22T22:02:35.745 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_46'] 2026-02-22T22:02:35.745 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:02:35.745 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T22:02:35.746 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:02:35.746 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sleep 2 2026-02-22T22:02:37.747 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:02:37.747 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ seq 0 3 2026-02-22T22:02:37.750 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:02:37.750 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_46 2026-02-22T22:02:37.750 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:0 2026-02-22T22:02:37.750 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:02:37.751 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T22:02:37.751 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:02:37.751 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:0 2026-02-22T22:02:37.751 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T22:02:37.751 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T22:02:37.751 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:02:37.751 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=0 2026-02-22T22:02:37.751 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:02:37.751 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T22:02:37.751 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:02:37.751 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=0 2026-02-22T22:02:37.752 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T22:02:37.752 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:02:37.752 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:02:37.752 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T22:02:37.752 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T22:02:37.752 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:02:37.752 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:0 2026-02-22T22:02:37.752 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:02:37.752 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T22:02:37.752 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:02:37.752 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T22:02:37.752 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:02:37.753 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:02:37.753 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T22:02:37.753 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=0 2026-02-22T22:02:37.753 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=0 2026-02-22T22:02:37.753 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:02:37.762 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:02:37.762 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:02:37.762 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:02:37.763 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_46 2026-02-22T22:02:37.809 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_46'] 2026-02-22T22:02:37.809 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:02:37.809 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T22:02:37.812 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:02:37.812 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_46 2026-02-22T22:02:37.812 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:1 2026-02-22T22:02:37.812 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:02:37.812 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T22:02:37.812 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:02:37.812 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:1 2026-02-22T22:02:37.812 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T22:02:37.813 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T22:02:37.813 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:02:37.813 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=1 2026-02-22T22:02:37.813 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:02:37.813 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T22:02:37.813 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:02:37.813 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=1 2026-02-22T22:02:37.813 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T22:02:37.813 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:02:37.813 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:1 2026-02-22T22:02:37.813 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T22:02:37.813 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T22:02:37.813 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:02:37.814 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:1 2026-02-22T22:02:37.814 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:02:37.814 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T22:02:37.814 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:02:37.814 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T22:02:37.814 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T22:02:37.814 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:02:37.814 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T22:02:37.814 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=1 2026-02-22T22:02:37.814 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=1 2026-02-22T22:02:37.814 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:02:37.822 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:02:37.823 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:02:37.823 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:02:37.823 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_46 2026-02-22T22:02:37.862 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_46'] 2026-02-22T22:02:37.863 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:02:37.863 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T22:02:37.866 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:02:37.866 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_46 2026-02-22T22:02:37.866 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:2 2026-02-22T22:02:37.866 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:02:37.866 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T22:02:37.866 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:02:37.866 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:2 2026-02-22T22:02:37.866 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T22:02:37.866 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T22:02:37.866 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:02:37.867 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=2 2026-02-22T22:02:37.867 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:02:37.867 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T22:02:37.867 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:02:37.867 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=2 2026-02-22T22:02:37.867 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T22:02:37.867 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:02:37.867 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:2 2026-02-22T22:02:37.867 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T22:02:37.867 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T22:02:37.867 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:02:37.867 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:2 2026-02-22T22:02:37.868 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:02:37.868 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T22:02:37.868 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:02:37.868 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T22:02:37.868 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T22:02:37.868 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:02:37.868 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T22:02:37.868 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=2 2026-02-22T22:02:37.868 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=2 2026-02-22T22:02:37.868 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:02:37.876 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:02:37.876 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:02:37.876 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:02:37.876 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_46 2026-02-22T22:02:37.919 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_46'] 2026-02-22T22:02:37.919 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:02:37.919 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T22:02:37.922 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:02:37.922 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_46 2026-02-22T22:02:37.922 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:3 2026-02-22T22:02:37.923 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:02:37.923 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T22:02:37.923 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T22:02:37.923 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:3 2026-02-22T22:02:37.923 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T22:02:37.923 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T22:02:37.923 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:02:37.923 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=3 2026-02-22T22:02:37.923 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T22:02:37.923 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T22:02:37.923 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:02:37.923 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=3 2026-02-22T22:02:37.924 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=3 2026-02-22T22:02:37.924 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T22:02:37.924 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:3 2026-02-22T22:02:37.924 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T22:02:37.924 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T22:02:37.924 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T22:02:37.924 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:3 2026-02-22T22:02:37.924 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:02:37.924 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T22:02:37.924 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:02:37.924 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=3 2026-02-22T22:02:37.924 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T22:02:37.925 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:02:37.925 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T22:02:37.925 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=3 2026-02-22T22:02:37.925 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=3 2026-02-22T22:02:37.925 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T22:02:37.932 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:02:37.932 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:02:37.932 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:02:37.932 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_46 2026-02-22T22:02:37.976 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_46'] 2026-02-22T22:02:37.976 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:02:37.976 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T22:02:37.977 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:02:37.977 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sleep 4 2026-02-22T22:02:41.978 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:02:41.979 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ seq 0 3 2026-02-22T22:02:41.982 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:02:41.982 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_46 2026-02-22T22:02:41.982 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:0 2026-02-22T22:02:41.982 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:02:41.982 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T22:02:41.982 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:02:41.982 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:0 2026-02-22T22:02:41.982 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T22:02:41.982 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T22:02:41.983 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:02:41.983 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=0 2026-02-22T22:02:41.983 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:02:41.983 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T22:02:41.983 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:02:41.983 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=0 2026-02-22T22:02:41.983 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T22:02:41.983 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:02:41.983 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:02:41.983 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T22:02:41.984 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T22:02:41.984 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:02:41.984 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:0 2026-02-22T22:02:41.984 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:02:41.984 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T22:02:41.984 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:02:41.984 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T22:02:41.984 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:02:41.984 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:02:41.984 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T22:02:41.985 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=0 2026-02-22T22:02:41.985 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=0 2026-02-22T22:02:41.985 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:02:41.996 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:02:41.996 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:02:41.996 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:02:41.996 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_46 2026-02-22T22:02:42.041 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_46'] 2026-02-22T22:02:42.042 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:02:42.042 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T22:02:42.045 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:02:42.045 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_46 2026-02-22T22:02:42.045 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:1 2026-02-22T22:02:42.045 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:02:42.046 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T22:02:42.046 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:02:42.046 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:1 2026-02-22T22:02:42.046 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T22:02:42.046 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T22:02:42.046 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:02:42.046 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=1 2026-02-22T22:02:42.046 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:02:42.046 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T22:02:42.046 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:02:42.046 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=1 2026-02-22T22:02:42.047 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T22:02:42.047 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:02:42.047 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:1 2026-02-22T22:02:42.047 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T22:02:42.047 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T22:02:42.047 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:02:42.047 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:1 2026-02-22T22:02:42.047 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:02:42.047 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T22:02:42.047 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:02:42.047 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T22:02:42.047 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T22:02:42.047 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:02:42.048 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T22:02:42.048 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=1 2026-02-22T22:02:42.048 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=1 2026-02-22T22:02:42.048 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:02:42.056 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:02:42.056 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:02:42.056 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:02:42.056 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_46 2026-02-22T22:02:42.099 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_46'] 2026-02-22T22:02:42.099 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:02:42.100 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T22:02:42.103 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:02:42.103 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_46 2026-02-22T22:02:42.103 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:2 2026-02-22T22:02:42.104 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:02:42.104 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T22:02:42.104 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:02:42.104 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:2 2026-02-22T22:02:42.104 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T22:02:42.104 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T22:02:42.104 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:02:42.104 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=2 2026-02-22T22:02:42.104 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:02:42.104 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T22:02:42.104 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:02:42.105 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=2 2026-02-22T22:02:42.105 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T22:02:42.105 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:02:42.105 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:2 2026-02-22T22:02:42.105 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T22:02:42.105 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T22:02:42.105 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:02:42.105 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:2 2026-02-22T22:02:42.105 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:02:42.105 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T22:02:42.105 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:02:42.105 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T22:02:42.106 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T22:02:42.106 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:02:42.106 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T22:02:42.106 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=2 2026-02-22T22:02:42.106 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=2 2026-02-22T22:02:42.106 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:02:42.113 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:02:42.114 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:02:42.114 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:02:42.114 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_46 2026-02-22T22:02:42.157 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_46'] 2026-02-22T22:02:42.157 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:02:42.157 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T22:02:42.160 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:02:42.161 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_46 2026-02-22T22:02:42.161 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:3 2026-02-22T22:02:42.161 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:02:42.161 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T22:02:42.161 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T22:02:42.161 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:3 2026-02-22T22:02:42.161 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T22:02:42.161 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T22:02:42.161 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:02:42.161 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=3 2026-02-22T22:02:42.162 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T22:02:42.162 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T22:02:42.162 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:02:42.162 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=3 2026-02-22T22:02:42.162 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=3 2026-02-22T22:02:42.162 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T22:02:42.162 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:3 2026-02-22T22:02:42.162 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T22:02:42.162 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T22:02:42.162 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T22:02:42.162 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:3 2026-02-22T22:02:42.163 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:02:42.163 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T22:02:42.163 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:02:42.163 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=3 2026-02-22T22:02:42.163 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T22:02:42.163 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:02:42.163 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T22:02:42.163 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=3 2026-02-22T22:02:42.163 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=3 2026-02-22T22:02:42.163 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T22:02:42.170 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:02:42.170 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:02:42.170 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:02:42.170 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_46 2026-02-22T22:02:42.214 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_46'] 2026-02-22T22:02:42.214 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:02:42.215 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T22:02:42.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:02:42.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sleep 8 2026-02-22T22:02:50.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:02:50.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ seq 0 3 2026-02-22T22:02:50.218 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:02:50.218 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_46 2026-02-22T22:02:50.218 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:0 2026-02-22T22:02:50.218 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:02:50.218 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T22:02:50.218 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:02:50.218 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:0 2026-02-22T22:02:50.218 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T22:02:50.218 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T22:02:50.218 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:02:50.218 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=0 2026-02-22T22:02:50.218 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:02:50.218 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T22:02:50.218 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:02:50.218 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=0 2026-02-22T22:02:50.218 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T22:02:50.219 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:02:50.219 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:02:50.219 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T22:02:50.219 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T22:02:50.219 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:02:50.219 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:0 2026-02-22T22:02:50.219 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:02:50.219 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T22:02:50.219 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:02:50.219 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T22:02:50.219 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:02:50.219 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:02:50.219 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T22:02:50.219 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=0 2026-02-22T22:02:50.219 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=0 2026-02-22T22:02:50.219 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:02:50.231 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:02:50.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:02:50.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:02:50.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_46 2026-02-22T22:02:50.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_46'] 2026-02-22T22:02:50.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:02:50.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T22:02:50.280 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:02:50.280 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_46 2026-02-22T22:02:50.280 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:1 2026-02-22T22:02:50.280 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:02:50.280 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T22:02:50.280 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:02:50.280 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:1 2026-02-22T22:02:50.280 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T22:02:50.280 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T22:02:50.280 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:02:50.280 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=1 2026-02-22T22:02:50.280 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:02:50.280 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T22:02:50.280 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:02:50.280 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=1 2026-02-22T22:02:50.280 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T22:02:50.280 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:02:50.280 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:1 2026-02-22T22:02:50.280 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T22:02:50.280 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T22:02:50.280 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:02:50.281 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:1 2026-02-22T22:02:50.281 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:02:50.281 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T22:02:50.281 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:02:50.281 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T22:02:50.281 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T22:02:50.281 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:02:50.281 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T22:02:50.281 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=1 2026-02-22T22:02:50.281 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=1 2026-02-22T22:02:50.281 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:02:50.292 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:02:50.292 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:02:50.292 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:02:50.292 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_46 2026-02-22T22:02:50.337 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_46'] 2026-02-22T22:02:50.337 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:02:50.337 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T22:02:50.340 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:02:50.341 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_46 2026-02-22T22:02:50.341 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:2 2026-02-22T22:02:50.341 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:02:50.341 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T22:02:50.341 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:02:50.341 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:2 2026-02-22T22:02:50.341 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T22:02:50.341 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T22:02:50.341 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:02:50.341 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=2 2026-02-22T22:02:50.341 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:02:50.342 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T22:02:50.342 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:02:50.342 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=2 2026-02-22T22:02:50.342 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T22:02:50.342 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:02:50.342 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:2 2026-02-22T22:02:50.342 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T22:02:50.342 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T22:02:50.342 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:02:50.342 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:2 2026-02-22T22:02:50.342 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:02:50.343 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T22:02:50.343 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:02:50.343 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T22:02:50.343 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T22:02:50.343 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:02:50.343 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T22:02:50.343 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=2 2026-02-22T22:02:50.343 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=2 2026-02-22T22:02:50.343 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:02:50.351 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:02:50.351 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:02:50.351 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:02:50.351 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_46 2026-02-22T22:02:50.395 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_46'] 2026-02-22T22:02:50.395 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:02:50.396 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T22:02:50.399 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:02:50.399 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_46 2026-02-22T22:02:50.399 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:3 2026-02-22T22:02:50.399 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:02:50.400 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T22:02:50.400 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T22:02:50.400 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:3 2026-02-22T22:02:50.400 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T22:02:50.400 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T22:02:50.400 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:02:50.400 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=3 2026-02-22T22:02:50.400 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T22:02:50.400 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T22:02:50.400 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:02:50.400 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=3 2026-02-22T22:02:50.401 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=3 2026-02-22T22:02:50.401 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T22:02:50.401 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:3 2026-02-22T22:02:50.401 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T22:02:50.401 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T22:02:50.401 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T22:02:50.401 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:3 2026-02-22T22:02:50.401 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:02:50.401 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T22:02:50.401 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:02:50.401 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=3 2026-02-22T22:02:50.401 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T22:02:50.402 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:02:50.402 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T22:02:50.402 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=3 2026-02-22T22:02:50.402 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=3 2026-02-22T22:02:50.402 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T22:02:50.409 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:02:50.410 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:02:50.410 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:02:50.410 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_46 2026-02-22T22:02:50.454 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_46'] 2026-02-22T22:02:50.454 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:02:50.455 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T22:02:50.455 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:02:50.455 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sleep 8 2026-02-22T22:02:58.457 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:02:58.457 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ seq 0 3 2026-02-22T22:02:58.460 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:02:58.460 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_46 2026-02-22T22:02:58.460 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:0 2026-02-22T22:02:58.461 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:02:58.461 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T22:02:58.461 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:02:58.461 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:0 2026-02-22T22:02:58.461 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T22:02:58.461 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T22:02:58.461 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:02:58.461 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=0 2026-02-22T22:02:58.461 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:02:58.461 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T22:02:58.461 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:02:58.461 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=0 2026-02-22T22:02:58.462 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T22:02:58.462 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:02:58.462 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:02:58.462 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T22:02:58.462 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T22:02:58.462 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:02:58.462 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:0 2026-02-22T22:02:58.462 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:02:58.462 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T22:02:58.462 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:02:58.462 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T22:02:58.462 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:02:58.463 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:02:58.463 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T22:02:58.463 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=0 2026-02-22T22:02:58.463 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=0 2026-02-22T22:02:58.463 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:02:58.471 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:02:58.472 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:02:58.472 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:02:58.472 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_46 2026-02-22T22:02:58.517 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_46'] 2026-02-22T22:02:58.517 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:02:58.517 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T22:02:58.521 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:02:58.521 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_46 2026-02-22T22:02:58.521 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:1 2026-02-22T22:02:58.521 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:02:58.521 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T22:02:58.521 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:02:58.522 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:1 2026-02-22T22:02:58.522 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T22:02:58.522 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T22:02:58.522 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:02:58.522 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=1 2026-02-22T22:02:58.522 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:02:58.522 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T22:02:58.522 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:02:58.522 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=1 2026-02-22T22:02:58.522 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T22:02:58.522 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:02:58.522 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:1 2026-02-22T22:02:58.523 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T22:02:58.523 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T22:02:58.523 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:02:58.523 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:1 2026-02-22T22:02:58.523 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:02:58.523 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T22:02:58.523 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:02:58.523 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T22:02:58.523 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T22:02:58.523 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:02:58.523 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T22:02:58.523 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=1 2026-02-22T22:02:58.524 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=1 2026-02-22T22:02:58.524 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:02:58.531 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:02:58.531 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:02:58.531 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:02:58.531 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_46 2026-02-22T22:02:58.574 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_46'] 2026-02-22T22:02:58.575 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:02:58.575 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T22:02:58.578 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:02:58.578 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_46 2026-02-22T22:02:58.578 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:2 2026-02-22T22:02:58.578 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:02:58.579 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T22:02:58.579 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:02:58.579 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:2 2026-02-22T22:02:58.579 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T22:02:58.579 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T22:02:58.579 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:02:58.579 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=2 2026-02-22T22:02:58.579 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:02:58.579 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T22:02:58.579 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:02:58.579 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=2 2026-02-22T22:02:58.579 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T22:02:58.580 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:02:58.580 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:2 2026-02-22T22:02:58.580 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T22:02:58.580 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T22:02:58.580 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:02:58.580 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:2 2026-02-22T22:02:58.580 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:02:58.580 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T22:02:58.580 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:02:58.580 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T22:02:58.580 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T22:02:58.581 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:02:58.581 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T22:02:58.581 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=2 2026-02-22T22:02:58.581 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=2 2026-02-22T22:02:58.581 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:02:58.588 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:02:58.588 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:02:58.588 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:02:58.588 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_46 2026-02-22T22:02:58.631 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_46'] 2026-02-22T22:02:58.632 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:02:58.632 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T22:02:58.635 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:02:58.635 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_46 2026-02-22T22:02:58.635 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:3 2026-02-22T22:02:58.635 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:02:58.635 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T22:02:58.635 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T22:02:58.635 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:3 2026-02-22T22:02:58.636 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T22:02:58.636 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T22:02:58.636 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:02:58.636 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=3 2026-02-22T22:02:58.636 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T22:02:58.636 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T22:02:58.636 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:02:58.636 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=3 2026-02-22T22:02:58.636 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=3 2026-02-22T22:02:58.636 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T22:02:58.636 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:3 2026-02-22T22:02:58.637 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T22:02:58.637 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T22:02:58.637 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T22:02:58.637 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:3 2026-02-22T22:02:58.637 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:02:58.637 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T22:02:58.637 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:02:58.637 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=3 2026-02-22T22:02:58.637 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T22:02:58.637 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:02:58.637 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T22:02:58.638 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=3 2026-02-22T22:02:58.638 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=3 2026-02-22T22:02:58.638 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T22:02:58.645 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:02:58.645 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:02:58.645 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:02:58.645 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_46 2026-02-22T22:02:58.688 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_46'] 2026-02-22T22:02:58.689 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:02:58.689 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T22:02:58.690 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:02:58.690 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sleep 8 2026-02-22T22:03:06.691 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:03:06.691 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ seq 0 3 2026-02-22T22:03:06.694 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:03:06.694 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_46 2026-02-22T22:03:06.694 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:0 2026-02-22T22:03:06.695 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:03:06.695 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T22:03:06.695 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:03:06.695 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:0 2026-02-22T22:03:06.695 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T22:03:06.695 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T22:03:06.695 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:03:06.695 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=0 2026-02-22T22:03:06.695 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:03:06.695 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T22:03:06.695 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:03:06.695 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=0 2026-02-22T22:03:06.696 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T22:03:06.696 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:03:06.696 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:03:06.696 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T22:03:06.696 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T22:03:06.696 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:03:06.696 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:0 2026-02-22T22:03:06.696 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:03:06.696 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T22:03:06.696 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:03:06.696 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T22:03:06.696 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:03:06.697 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:03:06.697 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T22:03:06.697 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=0 2026-02-22T22:03:06.697 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=0 2026-02-22T22:03:06.697 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:03:06.705 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:03:06.706 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:03:06.706 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:03:06.706 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_46 2026-02-22T22:03:06.752 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_46'] 2026-02-22T22:03:06.752 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:03:06.753 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T22:03:06.756 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:03:06.756 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_46 2026-02-22T22:03:06.756 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:1 2026-02-22T22:03:06.756 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:03:06.756 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T22:03:06.756 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:03:06.756 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:1 2026-02-22T22:03:06.756 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T22:03:06.757 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T22:03:06.757 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:03:06.757 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=1 2026-02-22T22:03:06.757 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:03:06.757 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T22:03:06.757 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:03:06.757 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=1 2026-02-22T22:03:06.757 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T22:03:06.757 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:03:06.757 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:1 2026-02-22T22:03:06.757 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T22:03:06.757 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T22:03:06.758 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:03:06.758 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:1 2026-02-22T22:03:06.758 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:03:06.758 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T22:03:06.758 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:03:06.758 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T22:03:06.758 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T22:03:06.758 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:03:06.758 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T22:03:06.758 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=1 2026-02-22T22:03:06.758 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=1 2026-02-22T22:03:06.758 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:03:06.766 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:03:06.766 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:03:06.767 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:03:06.767 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_46 2026-02-22T22:03:06.816 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ return 0 2026-02-22T22:03:06.817 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:03:06.817 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:03:06.817 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep -i Replaying 2026-02-22T22:03:06.818 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: "state": "Replaying" 2026-02-22T22:03:06.819 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_state=started 2026-02-22T22:03:06.819 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test started = started 2026-02-22T22:03:06.819 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:03:06.819 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_46 2026-02-22T22:03:06.820 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T22:03:06.820 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:03:06.820 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:03:06.820 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_46 2026-02-22T22:03:06.820 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = journal ']' 2026-02-22T22:03:06.820 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T22:03:06.820 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ mirror_image_snapshot cluster2 mirror image_46 2026-02-22T22:03:06.820 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:03:06.820 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:03:06.820 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_46 2026-02-22T22:03:06.820 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_46 2026-02-22T22:03:07.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Snapshot ID: 176 2026-02-22T22:03:07.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_46 2026-02-22T22:03:07.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T22:03:07.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:03:07.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:03:07.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_46 2026-02-22T22:03:07.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster2-mirror-image_46.status 2026-02-22T22:03:07.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster2-mirror-image_46.status 2026-02-22T22:03:07.218 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:03:07.218 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_log=/tmp/tmp.j44LZPowal/cluster2-mirror-image_46.status 2026-02-22T22:03:07.218 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster1-mirror-image_46.status 2026-02-22T22:03:07.219 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster1-mirror-image_46.status 2026-02-22T22:03:07.219 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:03:07.220 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_status_log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_46.status 2026-02-22T22:03:07.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_46 /tmp/tmp.j44LZPowal/cluster2-mirror-image_46.status 2026-02-22T22:03:07.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:03:07.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:03:07.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_46 2026-02-22T22:03:07.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster2-mirror-image_46.status 2026-02-22T22:03:07.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_46 --format xml 2026-02-22T22:03:07.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:03:07.248 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T22:03:07.249 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snapshot_id=176 2026-02-22T22:03:07.249 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T22:03:07.249 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:03:07.250 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.2 2026-02-22T22:03:07.451 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_46 /tmp/tmp.j44LZPowal/cluster1-mirror-image_46.status 2026-02-22T22:03:07.451 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:03:07.451 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:03:07.452 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_46 2026-02-22T22:03:07.452 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_46.status 2026-02-22T22:03:07.452 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_46 --format xml 2026-02-22T22:03:07.452 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:03:07.479 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T22:03:07.479 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T22:03:07.480 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T22:03:07.480 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T22:03:07.480 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T22:03:07.481 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T22:03:07.481 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 176 = '' 2026-02-22T22:03:07.481 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:03:07.481 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.4 2026-02-22T22:03:07.882 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_46 /tmp/tmp.j44LZPowal/cluster1-mirror-image_46.status 2026-02-22T22:03:07.882 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:03:07.882 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:03:07.882 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_46 2026-02-22T22:03:07.882 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_46.status 2026-02-22T22:03:07.883 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_46 --format xml 2026-02-22T22:03:07.883 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:03:07.912 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T22:03:07.912 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T22:03:07.913 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T22:03:07.913 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T22:03:07.913 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T22:03:07.913 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T22:03:07.913 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 176 = '' 2026-02-22T22:03:07.913 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:03:07.913 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.8 2026-02-22T22:03:08.715 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_46 /tmp/tmp.j44LZPowal/cluster1-mirror-image_46.status 2026-02-22T22:03:08.715 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:03:08.715 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:03:08.716 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_46 2026-02-22T22:03:08.716 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_46.status 2026-02-22T22:03:08.716 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_46 --format xml 2026-02-22T22:03:08.716 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:03:08.745 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T22:03:08.747 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id=176 2026-02-22T22:03:08.748 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 176 = 176 2026-02-22T22:03:08.748 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:03:08.748 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snap_present cluster1 mirror image_46 snap 2026-02-22T22:03:08.748 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:03:08.748 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:03:08.748 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_46 2026-02-22T22:03:08.748 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T22:03:08.748 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:03:08.748 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T22:03:08.748 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T22:03:09.748 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 info mirror/image_46@snap 2026-02-22T22:03:09.777 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_46': 2026-02-22T22:03:09.777 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 128 MiB in 32 objects 2026-02-22T22:03:09.778 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T22:03:09.778 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 2 2026-02-22T22:03:09.778 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 1227b57fb5e1 2026-02-22T22:03:09.778 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.1227b57fb5e1 2026-02-22T22:03:09.778 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T22:03:09.778 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock, non-primary 2026-02-22T22:03:09.778 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T22:03:09.778 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T22:03:09.778 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:55:33 2026 2026-02-22T22:03:09.778 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:55:33 2026 2026-02-22T22:03:09.779 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:55:33 2026 2026-02-22T22:03:09.779 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: protected: False 2026-02-22T22:03:09.779 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T22:03:09.779 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T22:03:09.779 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: 04845371-74eb-4487-a2bf-eec2351b191c 2026-02-22T22:03:09.779 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: false 2026-02-22T22:03:09.780 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:03:09.780 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ compare_image_snaps mirror image_46 snap 2026-02-22T22:03:09.780 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:03:09.780 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_46 2026-02-22T22:03:09.781 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T22:03:09.781 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local ret=0 2026-02-22T22:03:09.781 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local rmt_export=/tmp/tmp.j44LZPowal/cluster2-mirror-image_46.export 2026-02-22T22:03:09.781 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local loc_export=/tmp/tmp.j44LZPowal/cluster1-mirror-image_46.export 2026-02-22T22:03:09.781 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-image_46.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_46.export 2026-02-22T22:03:09.781 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror export image_46@snap /tmp/tmp.j44LZPowal/cluster2-mirror-image_46.export 2026-02-22T22:03:09.886 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:03:09.894 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 -p mirror export image_46@snap /tmp/tmp.j44LZPowal/cluster1-mirror-image_46.export 2026-02-22T22:03:10.020 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:03:10.027 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ cmp /tmp/tmp.j44LZPowal/cluster2-mirror-image_46.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_46.export 2026-02-22T22:03:10.077 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-image_46.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_46.export 2026-02-22T22:03:10.110 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:03:10.110 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:03:10.110 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_47 2026-02-22T22:03:10.110 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_snap cluster2 mirror image_47 snap 2026-02-22T22:03:10.110 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:03:10.111 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:03:10.111 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_47 2026-02-22T22:03:10.111 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T22:03:10.111 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror snap create image_47@snap --debug-rbd=20 --debug-journaler=20 2026-02-22T22:03:11.220 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_started cluster1 mirror image_47 2026-02-22T22:03:11.220 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:03:11.220 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:03:11.220 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_47 2026-02-22T22:03:11.220 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_state cluster1 mirror image_47 started 2026-02-22T22:03:11.220 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:03:11.220 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:03:11.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_47 2026-02-22T22:03:11.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=started 2026-02-22T22:03:11.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:03:11.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:03:11.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T22:03:12.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_replay_state cluster1 mirror image_47 started 2026-02-22T22:03:12.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:03:12.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:03:12.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_47 2026-02-22T22:03:12.222 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=started 2026-02-22T22:03:12.222 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_result 2026-02-22T22:03:12.222 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=stopped 2026-02-22T22:03:12.222 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_47 2026-02-22T22:03:12.222 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_instance=cluster1 2026-02-22T22:03:12.222 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:03:12.222 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:03:12.222 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance=cluster1 2026-02-22T22:03:12.222 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local loop_instance 2026-02-22T22:03:12.222 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:03:12.222 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sleep 0 2026-02-22T22:03:12.223 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ grep -i state 2026-02-22T22:03:12.225 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:03:12.225 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ seq 0 3 2026-02-22T22:03:12.226 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:03:12.226 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_47 2026-02-22T22:03:12.226 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:0 2026-02-22T22:03:12.226 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:03:12.226 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T22:03:12.226 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:03:12.226 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:0 2026-02-22T22:03:12.226 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T22:03:12.226 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T22:03:12.226 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:03:12.226 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=0 2026-02-22T22:03:12.226 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:03:12.227 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T22:03:12.227 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:03:12.227 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=0 2026-02-22T22:03:12.227 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T22:03:12.227 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:03:12.227 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:03:12.227 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T22:03:12.227 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T22:03:12.227 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:03:12.227 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:0 2026-02-22T22:03:12.227 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:03:12.227 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T22:03:12.227 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:03:12.227 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T22:03:12.228 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:03:12.228 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:03:12.228 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T22:03:12.228 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=0 2026-02-22T22:03:12.228 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=0 2026-02-22T22:03:12.228 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:03:12.236 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:03:12.236 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:03:12.236 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:03:12.237 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_47 2026-02-22T22:03:12.282 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_47'] 2026-02-22T22:03:12.282 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:03:12.283 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T22:03:12.285 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:03:12.286 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_47 2026-02-22T22:03:12.286 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:1 2026-02-22T22:03:12.286 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:03:12.286 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T22:03:12.286 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:03:12.286 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:1 2026-02-22T22:03:12.286 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T22:03:12.286 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T22:03:12.286 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:03:12.286 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=1 2026-02-22T22:03:12.286 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:03:12.287 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T22:03:12.287 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:03:12.287 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=1 2026-02-22T22:03:12.287 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T22:03:12.287 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:03:12.287 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:1 2026-02-22T22:03:12.287 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T22:03:12.287 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T22:03:12.287 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:03:12.287 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:1 2026-02-22T22:03:12.287 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:03:12.287 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T22:03:12.288 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:03:12.288 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T22:03:12.288 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T22:03:12.288 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:03:12.288 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T22:03:12.288 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=1 2026-02-22T22:03:12.288 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=1 2026-02-22T22:03:12.288 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:03:12.295 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:03:12.295 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:03:12.296 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:03:12.296 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_47 2026-02-22T22:03:12.348 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ return 0 2026-02-22T22:03:12.349 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:03:12.349 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:03:12.349 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep -i Replaying 2026-02-22T22:03:12.350 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: "state": "Replaying" 2026-02-22T22:03:12.351 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_state=started 2026-02-22T22:03:12.351 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test started = started 2026-02-22T22:03:12.351 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:03:12.352 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_47 2026-02-22T22:03:12.352 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T22:03:12.352 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:03:12.352 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:03:12.352 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_47 2026-02-22T22:03:12.352 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = journal ']' 2026-02-22T22:03:12.352 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T22:03:12.352 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ mirror_image_snapshot cluster2 mirror image_47 2026-02-22T22:03:12.352 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:03:12.352 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:03:12.352 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_47 2026-02-22T22:03:12.352 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_47 2026-02-22T22:03:13.220 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Snapshot ID: 178 2026-02-22T22:03:13.226 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_47 2026-02-22T22:03:13.227 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T22:03:13.227 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:03:13.227 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:03:13.227 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_47 2026-02-22T22:03:13.227 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster2-mirror-image_47.status 2026-02-22T22:03:13.228 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster2-mirror-image_47.status 2026-02-22T22:03:13.228 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:03:13.228 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_log=/tmp/tmp.j44LZPowal/cluster2-mirror-image_47.status 2026-02-22T22:03:13.229 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster1-mirror-image_47.status 2026-02-22T22:03:13.229 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster1-mirror-image_47.status 2026-02-22T22:03:13.229 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:03:13.230 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_status_log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_47.status 2026-02-22T22:03:13.230 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_47 /tmp/tmp.j44LZPowal/cluster2-mirror-image_47.status 2026-02-22T22:03:13.231 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:03:13.231 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:03:13.231 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_47 2026-02-22T22:03:13.231 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster2-mirror-image_47.status 2026-02-22T22:03:13.231 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_47 --format xml 2026-02-22T22:03:13.231 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:03:13.259 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T22:03:13.260 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snapshot_id=178 2026-02-22T22:03:13.261 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T22:03:13.261 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:03:13.261 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.2 2026-02-22T22:03:13.461 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_47 /tmp/tmp.j44LZPowal/cluster1-mirror-image_47.status 2026-02-22T22:03:13.461 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:03:13.461 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:03:13.461 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_47 2026-02-22T22:03:13.461 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_47.status 2026-02-22T22:03:13.461 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_47 --format xml 2026-02-22T22:03:13.461 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:03:13.491 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T22:03:13.492 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T22:03:13.493 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T22:03:13.493 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T22:03:13.493 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T22:03:13.494 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T22:03:13.494 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 178 = '' 2026-02-22T22:03:13.494 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:03:13.494 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.4 2026-02-22T22:03:13.895 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_47 /tmp/tmp.j44LZPowal/cluster1-mirror-image_47.status 2026-02-22T22:03:13.895 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:03:13.895 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:03:13.895 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_47 2026-02-22T22:03:13.895 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_47.status 2026-02-22T22:03:13.895 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_47 --format xml 2026-02-22T22:03:13.896 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:03:13.928 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T22:03:13.928 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T22:03:13.929 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T22:03:13.929 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T22:03:13.929 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T22:03:13.930 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T22:03:13.930 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 178 = '' 2026-02-22T22:03:13.930 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:03:13.930 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.8 2026-02-22T22:03:14.731 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_47 /tmp/tmp.j44LZPowal/cluster1-mirror-image_47.status 2026-02-22T22:03:14.731 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:03:14.731 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:03:14.731 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_47 2026-02-22T22:03:14.731 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_47.status 2026-02-22T22:03:14.731 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_47 --format xml 2026-02-22T22:03:14.731 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:03:14.763 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T22:03:14.765 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id=178 2026-02-22T22:03:14.765 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 178 = 178 2026-02-22T22:03:14.765 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:03:14.765 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snap_present cluster1 mirror image_47 snap 2026-02-22T22:03:14.765 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:03:14.765 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:03:14.765 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_47 2026-02-22T22:03:14.765 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T22:03:14.765 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:03:14.766 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T22:03:14.766 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T22:03:15.766 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 info mirror/image_47@snap 2026-02-22T22:03:15.795 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_47': 2026-02-22T22:03:15.795 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 128 MiB in 32 objects 2026-02-22T22:03:15.795 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T22:03:15.795 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 2 2026-02-22T22:03:15.795 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 126018cbb057 2026-02-22T22:03:15.796 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.126018cbb057 2026-02-22T22:03:15.796 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T22:03:15.796 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock, non-primary 2026-02-22T22:03:15.796 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T22:03:15.796 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T22:03:15.796 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:55:44 2026 2026-02-22T22:03:15.796 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:55:44 2026 2026-02-22T22:03:15.796 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:55:44 2026 2026-02-22T22:03:15.796 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: protected: False 2026-02-22T22:03:15.796 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T22:03:15.796 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T22:03:15.797 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: 1c0dd8a5-46f9-44d8-9de4-48e37b2678f7 2026-02-22T22:03:15.797 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: false 2026-02-22T22:03:15.798 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:03:15.798 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ compare_image_snaps mirror image_47 snap 2026-02-22T22:03:15.798 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:03:15.799 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_47 2026-02-22T22:03:15.799 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T22:03:15.799 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local ret=0 2026-02-22T22:03:15.799 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local rmt_export=/tmp/tmp.j44LZPowal/cluster2-mirror-image_47.export 2026-02-22T22:03:15.799 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local loc_export=/tmp/tmp.j44LZPowal/cluster1-mirror-image_47.export 2026-02-22T22:03:15.799 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-image_47.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_47.export 2026-02-22T22:03:15.799 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror export image_47@snap /tmp/tmp.j44LZPowal/cluster2-mirror-image_47.export 2026-02-22T22:03:15.903 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:03:15.911 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 -p mirror export image_47@snap /tmp/tmp.j44LZPowal/cluster1-mirror-image_47.export 2026-02-22T22:03:16.031 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:03:16.038 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ cmp /tmp/tmp.j44LZPowal/cluster2-mirror-image_47.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_47.export 2026-02-22T22:03:16.088 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-image_47.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_47.export 2026-02-22T22:03:16.122 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:03:16.122 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:03:16.122 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_48 2026-02-22T22:03:16.123 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_snap cluster2 mirror image_48 snap 2026-02-22T22:03:16.123 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:03:16.123 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:03:16.123 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_48 2026-02-22T22:03:16.123 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T22:03:16.123 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror snap create image_48@snap --debug-rbd=20 --debug-journaler=20 2026-02-22T22:03:17.224 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_started cluster1 mirror image_48 2026-02-22T22:03:17.224 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:03:17.225 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:03:17.225 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_48 2026-02-22T22:03:17.225 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_state cluster1 mirror image_48 started 2026-02-22T22:03:17.225 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:03:17.225 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:03:17.225 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_48 2026-02-22T22:03:17.225 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=started 2026-02-22T22:03:17.225 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:03:17.225 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:03:17.226 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T22:03:18.224 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_replay_state cluster1 mirror image_48 started 2026-02-22T22:03:18.225 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:03:18.225 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:03:18.225 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_48 2026-02-22T22:03:18.225 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=started 2026-02-22T22:03:18.225 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_result 2026-02-22T22:03:18.225 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=stopped 2026-02-22T22:03:18.225 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_48 2026-02-22T22:03:18.225 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_instance=cluster1 2026-02-22T22:03:18.225 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ grep -i state 2026-02-22T22:03:18.225 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:03:18.225 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:03:18.226 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance=cluster1 2026-02-22T22:03:18.226 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local loop_instance 2026-02-22T22:03:18.226 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:03:18.226 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sleep 0 2026-02-22T22:03:18.226 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:03:18.226 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ seq 0 3 2026-02-22T22:03:18.227 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:03:18.228 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_48 2026-02-22T22:03:18.228 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:0 2026-02-22T22:03:18.228 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:03:18.228 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T22:03:18.228 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:03:18.228 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:0 2026-02-22T22:03:18.228 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T22:03:18.228 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T22:03:18.228 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:03:18.228 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=0 2026-02-22T22:03:18.228 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:03:18.228 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T22:03:18.228 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:03:18.228 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=0 2026-02-22T22:03:18.228 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T22:03:18.228 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:03:18.228 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:03:18.228 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T22:03:18.228 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T22:03:18.228 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:03:18.228 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:0 2026-02-22T22:03:18.228 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:03:18.228 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T22:03:18.228 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:03:18.228 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T22:03:18.228 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:03:18.228 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:03:18.228 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T22:03:18.228 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=0 2026-02-22T22:03:18.229 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=0 2026-02-22T22:03:18.229 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:03:18.240 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:03:18.240 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:03:18.240 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:03:18.240 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_48 2026-02-22T22:03:18.286 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_48'] 2026-02-22T22:03:18.286 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:03:18.287 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T22:03:18.292 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:03:18.292 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_48 2026-02-22T22:03:18.293 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:1 2026-02-22T22:03:18.293 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:03:18.293 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T22:03:18.293 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:03:18.293 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:1 2026-02-22T22:03:18.294 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T22:03:18.294 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T22:03:18.294 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:03:18.294 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=1 2026-02-22T22:03:18.294 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:03:18.294 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T22:03:18.295 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:03:18.295 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=1 2026-02-22T22:03:18.295 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T22:03:18.295 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:03:18.295 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:1 2026-02-22T22:03:18.295 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T22:03:18.296 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T22:03:18.296 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:03:18.296 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:1 2026-02-22T22:03:18.296 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:03:18.296 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T22:03:18.296 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:03:18.297 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T22:03:18.297 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T22:03:18.297 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:03:18.297 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T22:03:18.297 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=1 2026-02-22T22:03:18.297 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=1 2026-02-22T22:03:18.297 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:03:18.301 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:03:18.301 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:03:18.301 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:03:18.301 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_48 2026-02-22T22:03:18.352 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ return 0 2026-02-22T22:03:18.353 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:03:18.353 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:03:18.353 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep -i Replaying 2026-02-22T22:03:18.354 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: "state": "Replaying" 2026-02-22T22:03:18.355 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_state=started 2026-02-22T22:03:18.355 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test started = started 2026-02-22T22:03:18.355 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:03:18.356 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_48 2026-02-22T22:03:18.356 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T22:03:18.356 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:03:18.356 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:03:18.356 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_48 2026-02-22T22:03:18.356 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = journal ']' 2026-02-22T22:03:18.356 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T22:03:18.356 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ mirror_image_snapshot cluster2 mirror image_48 2026-02-22T22:03:18.356 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:03:18.356 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:03:18.356 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_48 2026-02-22T22:03:18.357 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_48 2026-02-22T22:03:19.227 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Snapshot ID: 180 2026-02-22T22:03:19.233 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_48 2026-02-22T22:03:19.233 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T22:03:19.233 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:03:19.233 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:03:19.233 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_48 2026-02-22T22:03:19.233 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster2-mirror-image_48.status 2026-02-22T22:03:19.233 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster2-mirror-image_48.status 2026-02-22T22:03:19.233 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:03:19.235 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_log=/tmp/tmp.j44LZPowal/cluster2-mirror-image_48.status 2026-02-22T22:03:19.235 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster1-mirror-image_48.status 2026-02-22T22:03:19.235 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster1-mirror-image_48.status 2026-02-22T22:03:19.236 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:03:19.237 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_status_log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_48.status 2026-02-22T22:03:19.237 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_48 /tmp/tmp.j44LZPowal/cluster2-mirror-image_48.status 2026-02-22T22:03:19.237 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:03:19.237 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:03:19.237 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_48 2026-02-22T22:03:19.237 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster2-mirror-image_48.status 2026-02-22T22:03:19.237 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_48 --format xml 2026-02-22T22:03:19.237 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:03:19.264 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T22:03:19.265 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snapshot_id=180 2026-02-22T22:03:19.266 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T22:03:19.266 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:03:19.266 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.2 2026-02-22T22:03:19.467 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_48 /tmp/tmp.j44LZPowal/cluster1-mirror-image_48.status 2026-02-22T22:03:19.467 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:03:19.468 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:03:19.468 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_48 2026-02-22T22:03:19.468 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_48.status 2026-02-22T22:03:19.468 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_48 --format xml 2026-02-22T22:03:19.468 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:03:19.496 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T22:03:19.496 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T22:03:19.497 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T22:03:19.497 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T22:03:19.497 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T22:03:19.498 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T22:03:19.498 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 180 = '' 2026-02-22T22:03:19.498 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:03:19.498 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.4 2026-02-22T22:03:19.899 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_48 /tmp/tmp.j44LZPowal/cluster1-mirror-image_48.status 2026-02-22T22:03:19.899 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:03:19.899 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:03:19.899 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_48 2026-02-22T22:03:19.900 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_48.status 2026-02-22T22:03:19.900 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_48 --format xml 2026-02-22T22:03:19.900 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:03:19.928 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T22:03:19.928 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T22:03:19.929 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T22:03:19.929 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T22:03:19.929 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T22:03:19.930 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T22:03:19.930 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 180 = '' 2026-02-22T22:03:19.930 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:03:19.930 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.8 2026-02-22T22:03:20.731 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_48 /tmp/tmp.j44LZPowal/cluster1-mirror-image_48.status 2026-02-22T22:03:20.731 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:03:20.731 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:03:20.731 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_48 2026-02-22T22:03:20.731 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_48.status 2026-02-22T22:03:20.732 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_48 --format xml 2026-02-22T22:03:20.732 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:03:20.761 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T22:03:20.763 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id=180 2026-02-22T22:03:20.763 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 180 = 180 2026-02-22T22:03:20.763 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:03:20.763 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snap_present cluster1 mirror image_48 snap 2026-02-22T22:03:20.763 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:03:20.763 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:03:20.764 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_48 2026-02-22T22:03:20.764 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T22:03:20.764 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:03:20.764 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T22:03:20.764 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T22:03:21.763 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 info mirror/image_48@snap 2026-02-22T22:03:21.791 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_48': 2026-02-22T22:03:21.791 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 128 MiB in 32 objects 2026-02-22T22:03:21.791 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T22:03:21.791 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 2 2026-02-22T22:03:21.791 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 126fb0457ca8 2026-02-22T22:03:21.791 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.126fb0457ca8 2026-02-22T22:03:21.791 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T22:03:21.791 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock, non-primary 2026-02-22T22:03:21.791 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T22:03:21.791 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T22:03:21.791 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:55:55 2026 2026-02-22T22:03:21.791 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:55:55 2026 2026-02-22T22:03:21.792 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:55:55 2026 2026-02-22T22:03:21.792 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: protected: False 2026-02-22T22:03:21.792 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T22:03:21.792 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T22:03:21.792 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: 073c2a09-55c1-4f22-ad52-735988b4ed7b 2026-02-22T22:03:21.792 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: false 2026-02-22T22:03:21.793 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:03:21.793 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ compare_image_snaps mirror image_48 snap 2026-02-22T22:03:21.793 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:03:21.793 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_48 2026-02-22T22:03:21.793 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T22:03:21.793 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local ret=0 2026-02-22T22:03:21.793 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local rmt_export=/tmp/tmp.j44LZPowal/cluster2-mirror-image_48.export 2026-02-22T22:03:21.793 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local loc_export=/tmp/tmp.j44LZPowal/cluster1-mirror-image_48.export 2026-02-22T22:03:21.793 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-image_48.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_48.export 2026-02-22T22:03:21.794 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror export image_48@snap /tmp/tmp.j44LZPowal/cluster2-mirror-image_48.export 2026-02-22T22:03:21.900 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:03:21.908 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 -p mirror export image_48@snap /tmp/tmp.j44LZPowal/cluster1-mirror-image_48.export 2026-02-22T22:03:22.031 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:03:22.038 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ cmp /tmp/tmp.j44LZPowal/cluster2-mirror-image_48.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_48.export 2026-02-22T22:03:22.087 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-image_48.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_48.export 2026-02-22T22:03:22.123 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:03:22.123 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:03:22.123 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_49 2026-02-22T22:03:22.123 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_snap cluster2 mirror image_49 snap 2026-02-22T22:03:22.123 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:03:22.123 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:03:22.123 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_49 2026-02-22T22:03:22.123 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T22:03:22.123 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror snap create image_49@snap --debug-rbd=20 --debug-journaler=20 2026-02-22T22:03:23.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_started cluster1 mirror image_49 2026-02-22T22:03:23.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:03:23.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:03:23.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_49 2026-02-22T22:03:23.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_state cluster1 mirror image_49 started 2026-02-22T22:03:23.233 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:03:23.233 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:03:23.233 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_49 2026-02-22T22:03:23.233 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=started 2026-02-22T22:03:23.233 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:03:23.233 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:03:23.233 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T22:03:24.234 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_replay_state cluster1 mirror image_49 started 2026-02-22T22:03:24.235 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:03:24.235 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:03:24.235 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_49 2026-02-22T22:03:24.235 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=started 2026-02-22T22:03:24.235 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_result 2026-02-22T22:03:24.235 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=stopped 2026-02-22T22:03:24.235 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_49 2026-02-22T22:03:24.235 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ grep -i state 2026-02-22T22:03:24.236 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_instance=cluster1 2026-02-22T22:03:24.236 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:03:24.236 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:03:24.236 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance=cluster1 2026-02-22T22:03:24.236 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local loop_instance 2026-02-22T22:03:24.236 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:03:24.236 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sleep 0 2026-02-22T22:03:24.239 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:03:24.239 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ seq 0 3 2026-02-22T22:03:24.239 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:03:24.239 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_49 2026-02-22T22:03:24.239 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:0 2026-02-22T22:03:24.239 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:03:24.239 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T22:03:24.239 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:03:24.239 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:0 2026-02-22T22:03:24.240 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T22:03:24.240 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T22:03:24.240 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:03:24.240 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=0 2026-02-22T22:03:24.240 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:03:24.240 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T22:03:24.240 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:03:24.240 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=0 2026-02-22T22:03:24.240 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T22:03:24.240 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:03:24.240 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:03:24.240 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T22:03:24.240 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T22:03:24.241 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:03:24.241 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:0 2026-02-22T22:03:24.241 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:03:24.241 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T22:03:24.241 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:03:24.241 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T22:03:24.241 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:03:24.241 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:03:24.241 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T22:03:24.241 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=0 2026-02-22T22:03:24.241 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=0 2026-02-22T22:03:24.241 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:03:24.250 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:03:24.250 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:03:24.250 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:03:24.250 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_49 2026-02-22T22:03:24.301 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ return 0 2026-02-22T22:03:24.301 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:03:24.301 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:03:24.301 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep -i Replaying 2026-02-22T22:03:24.302 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: "state": "Replaying" 2026-02-22T22:03:24.304 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_state=started 2026-02-22T22:03:24.304 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test started = started 2026-02-22T22:03:24.305 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:03:24.305 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_49 2026-02-22T22:03:24.305 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T22:03:24.305 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:03:24.305 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:03:24.305 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_49 2026-02-22T22:03:24.305 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = journal ']' 2026-02-22T22:03:24.305 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T22:03:24.306 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ mirror_image_snapshot cluster2 mirror image_49 2026-02-22T22:03:24.306 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:03:24.306 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:03:24.306 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_49 2026-02-22T22:03:24.306 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_49 2026-02-22T22:03:25.236 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Snapshot ID: 182 2026-02-22T22:03:25.242 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_49 2026-02-22T22:03:25.242 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T22:03:25.242 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:03:25.242 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:03:25.242 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_49 2026-02-22T22:03:25.242 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster2-mirror-image_49.status 2026-02-22T22:03:25.243 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster2-mirror-image_49.status 2026-02-22T22:03:25.243 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:03:25.244 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_log=/tmp/tmp.j44LZPowal/cluster2-mirror-image_49.status 2026-02-22T22:03:25.244 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster1-mirror-image_49.status 2026-02-22T22:03:25.244 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster1-mirror-image_49.status 2026-02-22T22:03:25.244 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:03:25.245 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_status_log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_49.status 2026-02-22T22:03:25.245 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_49 /tmp/tmp.j44LZPowal/cluster2-mirror-image_49.status 2026-02-22T22:03:25.246 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:03:25.246 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:03:25.246 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_49 2026-02-22T22:03:25.246 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster2-mirror-image_49.status 2026-02-22T22:03:25.246 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_49 --format xml 2026-02-22T22:03:25.246 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:03:25.273 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T22:03:25.275 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snapshot_id=182 2026-02-22T22:03:25.275 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T22:03:25.275 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:03:25.275 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.2 2026-02-22T22:03:25.476 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_49 /tmp/tmp.j44LZPowal/cluster1-mirror-image_49.status 2026-02-22T22:03:25.476 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:03:25.476 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:03:25.477 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_49 2026-02-22T22:03:25.477 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_49.status 2026-02-22T22:03:25.477 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_49 --format xml 2026-02-22T22:03:25.477 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:03:25.507 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T22:03:25.507 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T22:03:25.508 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T22:03:25.508 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T22:03:25.508 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T22:03:25.509 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T22:03:25.509 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 182 = '' 2026-02-22T22:03:25.509 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:03:25.509 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.4 2026-02-22T22:03:25.910 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_49 /tmp/tmp.j44LZPowal/cluster1-mirror-image_49.status 2026-02-22T22:03:25.910 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:03:25.911 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:03:25.911 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_49 2026-02-22T22:03:25.911 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_49.status 2026-02-22T22:03:25.911 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_49 --format xml 2026-02-22T22:03:25.911 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:03:25.942 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T22:03:25.943 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T22:03:25.944 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T22:03:25.944 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T22:03:25.945 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T22:03:25.945 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T22:03:25.945 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 182 = '' 2026-02-22T22:03:25.945 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:03:25.945 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.8 2026-02-22T22:03:26.745 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_49 /tmp/tmp.j44LZPowal/cluster1-mirror-image_49.status 2026-02-22T22:03:26.745 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:03:26.745 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:03:26.745 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_49 2026-02-22T22:03:26.745 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_49.status 2026-02-22T22:03:26.745 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_49 --format xml 2026-02-22T22:03:26.745 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:03:26.775 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T22:03:26.776 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id=182 2026-02-22T22:03:26.776 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 182 = 182 2026-02-22T22:03:26.776 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:03:26.776 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snap_present cluster1 mirror image_49 snap 2026-02-22T22:03:26.776 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:03:26.776 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:03:26.776 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_49 2026-02-22T22:03:26.776 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T22:03:26.777 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:03:26.777 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T22:03:26.777 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T22:03:27.778 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 info mirror/image_49@snap 2026-02-22T22:03:27.806 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_49': 2026-02-22T22:03:27.806 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 128 MiB in 32 objects 2026-02-22T22:03:27.806 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T22:03:27.806 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 2 2026-02-22T22:03:27.806 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 119a2ae6809d 2026-02-22T22:03:27.806 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.119a2ae6809d 2026-02-22T22:03:27.806 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T22:03:27.806 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock, non-primary 2026-02-22T22:03:27.806 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T22:03:27.806 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T22:03:27.806 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:56:10 2026 2026-02-22T22:03:27.806 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:56:10 2026 2026-02-22T22:03:27.806 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:56:10 2026 2026-02-22T22:03:27.806 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: protected: False 2026-02-22T22:03:27.806 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T22:03:27.806 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T22:03:27.806 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: 61a34d68-de2f-48ac-99bc-627b57103d64 2026-02-22T22:03:27.806 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: false 2026-02-22T22:03:27.807 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:03:27.808 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ compare_image_snaps mirror image_49 snap 2026-02-22T22:03:27.808 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:03:27.808 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_49 2026-02-22T22:03:27.808 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T22:03:27.808 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local ret=0 2026-02-22T22:03:27.808 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local rmt_export=/tmp/tmp.j44LZPowal/cluster2-mirror-image_49.export 2026-02-22T22:03:27.808 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local loc_export=/tmp/tmp.j44LZPowal/cluster1-mirror-image_49.export 2026-02-22T22:03:27.808 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-image_49.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_49.export 2026-02-22T22:03:27.808 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror export image_49@snap /tmp/tmp.j44LZPowal/cluster2-mirror-image_49.export 2026-02-22T22:03:27.916 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:03:27.922 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 -p mirror export image_49@snap /tmp/tmp.j44LZPowal/cluster1-mirror-image_49.export 2026-02-22T22:03:28.049 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:03:28.057 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ cmp /tmp/tmp.j44LZPowal/cluster2-mirror-image_49.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_49.export 2026-02-22T22:03:28.107 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-image_49.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_49.export 2026-02-22T22:03:28.142 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:03:28.142 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:03:28.142 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_50 2026-02-22T22:03:28.142 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_snap cluster2 mirror image_50 snap 2026-02-22T22:03:28.143 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:03:28.143 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:03:28.143 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_50 2026-02-22T22:03:28.143 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T22:03:28.143 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror snap create image_50@snap --debug-rbd=20 --debug-journaler=20 2026-02-22T22:03:29.241 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_started cluster1 mirror image_50 2026-02-22T22:03:29.241 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:03:29.241 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:03:29.241 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_50 2026-02-22T22:03:29.241 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_replay_state cluster1 mirror image_50 started 2026-02-22T22:03:29.241 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:03:29.241 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:03:29.241 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_50 2026-02-22T22:03:29.241 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=started 2026-02-22T22:03:29.241 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:03:29.241 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:03:29.241 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T22:03:30.242 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_replay_state cluster1 mirror image_50 started 2026-02-22T22:03:30.242 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:03:30.242 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:03:30.242 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_50 2026-02-22T22:03:30.243 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=started 2026-02-22T22:03:30.243 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_result 2026-02-22T22:03:30.243 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=stopped 2026-02-22T22:03:30.247 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_50 2026-02-22T22:03:30.248 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_instance=cluster1 2026-02-22T22:03:30.248 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:03:30.248 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ grep -i state 2026-02-22T22:03:30.248 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:03:30.248 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance=cluster1 2026-02-22T22:03:30.248 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local loop_instance 2026-02-22T22:03:30.248 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:03:30.248 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sleep 0 2026-02-22T22:03:30.249 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:03:30.249 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ seq 0 3 2026-02-22T22:03:30.249 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:03:30.249 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_50 2026-02-22T22:03:30.249 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:0 2026-02-22T22:03:30.249 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:03:30.249 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T22:03:30.250 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:03:30.250 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:0 2026-02-22T22:03:30.250 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T22:03:30.250 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T22:03:30.250 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:03:30.250 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=0 2026-02-22T22:03:30.250 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:03:30.250 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T22:03:30.250 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:03:30.250 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=0 2026-02-22T22:03:30.250 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T22:03:30.251 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:03:30.251 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:03:30.251 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T22:03:30.251 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T22:03:30.251 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:03:30.251 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:0 2026-02-22T22:03:30.251 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:03:30.251 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T22:03:30.251 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:03:30.251 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=0 2026-02-22T22:03:30.251 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:03:30.251 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:03:30.252 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T22:03:30.252 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=0 2026-02-22T22:03:30.252 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=0 2026-02-22T22:03:30.252 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:03:30.258 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:03:30.258 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:03:30.258 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:03:30.258 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_50 2026-02-22T22:03:30.304 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_50'] 2026-02-22T22:03:30.305 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:03:30.305 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T22:03:30.311 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:03:30.311 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_50 2026-02-22T22:03:30.311 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:1 2026-02-22T22:03:30.311 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:03:30.312 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T22:03:30.312 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:03:30.312 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:1 2026-02-22T22:03:30.312 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T22:03:30.312 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T22:03:30.312 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:03:30.312 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=1 2026-02-22T22:03:30.313 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:03:30.313 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T22:03:30.313 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:03:30.313 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=1 2026-02-22T22:03:30.313 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T22:03:30.313 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:03:30.313 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:1 2026-02-22T22:03:30.314 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T22:03:30.314 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T22:03:30.314 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:03:30.314 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:1 2026-02-22T22:03:30.314 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:03:30.314 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T22:03:30.314 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:03:30.314 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=1 2026-02-22T22:03:30.315 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T22:03:30.315 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:03:30.315 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T22:03:30.315 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=1 2026-02-22T22:03:30.315 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=1 2026-02-22T22:03:30.315 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:03:30.318 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:03:30.318 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:03:30.318 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:03:30.318 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_50 2026-02-22T22:03:30.363 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:Invalid command: unused arguments: ['mirror/image_50'] 2026-02-22T22:03:30.363 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:03:30.363 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:admin_socket: invalid command 2026-02-22T22:03:30.364 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:03:30.364 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_50 2026-02-22T22:03:30.364 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1:2 2026-02-22T22:03:30.364 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ shift 2026-02-22T22:03:30.364 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance 2026-02-22T22:03:30.364 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:03:30.364 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local val=cluster1:2 2026-02-22T22:03:30.364 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster_var_name=cluster 2026-02-22T22:03:30.364 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local instance_var_name=instance 2026-02-22T22:03:30.364 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:03:30.364 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=2 2026-02-22T22:03:30.364 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:03:30.364 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval cluster=cluster1 2026-02-22T22:03:30.364 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:03:30.364 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ eval instance=2 2026-02-22T22:03:30.365 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T22:03:30.365 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:03:30.365 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local local_cluster=cluster1:2 2026-02-22T22:03:30.365 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster=cluster1 2026-02-22T22:03:30.365 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance 2026-02-22T22:03:30.365 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:03:30.365 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local val=cluster1:2 2026-02-22T22:03:30.365 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:03:30.365 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ local instance_var_name=instance 2026-02-22T22:03:30.365 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ cluster=cluster1 2026-02-22T22:03:30.365 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ instance=2 2026-02-22T22:03:30.365 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T22:03:30.365 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:03:30.365 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ local_cluster=cluster1 2026-02-22T22:03:30.365 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ eval instance=2 2026-02-22T22:03:30.365 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ instance=2 2026-02-22T22:03:30.365 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:03:30.376 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:03:30.376 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:03:30.376 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:03:30.376 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_50 2026-02-22T22:03:30.427 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ return 0 2026-02-22T22:03:30.427 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:03:30.427 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:03:30.427 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ grep -i Replaying 2026-02-22T22:03:30.428 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: "state": "Replaying" 2026-02-22T22:03:30.429 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_state=started 2026-02-22T22:03:30.429 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test started = started 2026-02-22T22:03:30.429 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:03:30.429 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_50 2026-02-22T22:03:30.429 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T22:03:30.429 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:03:30.429 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:03:30.429 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_50 2026-02-22T22:03:30.429 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = journal ']' 2026-02-22T22:03:30.429 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T22:03:30.429 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ mirror_image_snapshot cluster2 mirror image_50 2026-02-22T22:03:30.429 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:03:30.429 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:03:30.429 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_50 2026-02-22T22:03:30.429 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_50 2026-02-22T22:03:31.246 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Snapshot ID: 184 2026-02-22T22:03:31.251 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_50 2026-02-22T22:03:31.251 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_cluster=cluster1 2026-02-22T22:03:31.251 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:03:31.251 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:03:31.251 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_50 2026-02-22T22:03:31.251 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster2-mirror-image_50.status 2026-02-22T22:03:31.252 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster2-mirror-image_50.status 2026-02-22T22:03:31.252 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:03:31.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local status_log=/tmp/tmp.j44LZPowal/cluster2-mirror-image_50.status 2026-02-22T22:03:31.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ mkfname cluster1-mirror-image_50.status 2026-02-22T22:03:31.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ echo cluster1-mirror-image_50.status 2026-02-22T22:03:31.254 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:03:31.255 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local local_status_log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_50.status 2026-02-22T22:03:31.255 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_50 /tmp/tmp.j44LZPowal/cluster2-mirror-image_50.status 2026-02-22T22:03:31.255 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:03:31.255 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:03:31.255 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_50 2026-02-22T22:03:31.255 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster2-mirror-image_50.status 2026-02-22T22:03:31.255 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_50 --format xml 2026-02-22T22:03:31.255 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:03:31.282 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T22:03:31.283 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snapshot_id=184 2026-02-22T22:03:31.283 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T22:03:31.283 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:03:31.283 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.2 2026-02-22T22:03:31.484 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_50 /tmp/tmp.j44LZPowal/cluster1-mirror-image_50.status 2026-02-22T22:03:31.485 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:03:31.485 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:03:31.485 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_50 2026-02-22T22:03:31.485 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_50.status 2026-02-22T22:03:31.485 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_50 --format xml 2026-02-22T22:03:31.485 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:03:31.514 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T22:03:31.514 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T22:03:31.515 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T22:03:31.515 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T22:03:31.515 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T22:03:31.516 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T22:03:31.516 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 184 = '' 2026-02-22T22:03:31.516 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:03:31.516 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.4 2026-02-22T22:03:31.917 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_50 /tmp/tmp.j44LZPowal/cluster1-mirror-image_50.status 2026-02-22T22:03:31.917 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:03:31.917 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:03:31.917 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_50 2026-02-22T22:03:31.917 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_50.status 2026-02-22T22:03:31.918 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_50 --format xml 2026-02-22T22:03:31.918 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:03:31.950 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T22:03:31.951 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T22:03:31.951 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:-:1.1: Document is empty 2026-02-22T22:03:31.951 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: 2026-02-22T22:03:31.951 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:^ 2026-02-22T22:03:31.952 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id= 2026-02-22T22:03:31.952 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 184 = '' 2026-02-22T22:03:31.952 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:03:31.952 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.8 2026-02-22T22:03:32.754 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_50 /tmp/tmp.j44LZPowal/cluster1-mirror-image_50.status 2026-02-22T22:03:32.755 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:03:32.755 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:03:32.755 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_50 2026-02-22T22:03:32.755 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local log=/tmp/tmp.j44LZPowal/cluster1-mirror-image_50.status 2026-02-22T22:03:32.755 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_50 --format xml 2026-02-22T22:03:32.755 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:03:32.787 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T22:03:32.788 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local primary_snapshot_id=184 2026-02-22T22:03:32.788 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 184 = 184 2026-02-22T22:03:32.788 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:03:32.788 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_snap_present cluster1 mirror image_50 snap 2026-02-22T22:03:32.788 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:03:32.788 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:03:32.788 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_50 2026-02-22T22:03:32.788 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T22:03:32.788 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:03:32.788 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T22:03:32.788 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T22:03:33.789 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 info mirror/image_50@snap 2026-02-22T22:03:33.818 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_50': 2026-02-22T22:03:33.818 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 128 MiB in 32 objects 2026-02-22T22:03:33.818 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T22:03:33.819 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 2 2026-02-22T22:03:33.819 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 1227fb031251 2026-02-22T22:03:33.819 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.1227fb031251 2026-02-22T22:03:33.819 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T22:03:33.819 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock, non-primary 2026-02-22T22:03:33.819 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T22:03:33.819 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T22:03:33.819 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 21:56:19 2026 2026-02-22T22:03:33.820 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 21:56:19 2026 2026-02-22T22:03:33.820 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 21:56:19 2026 2026-02-22T22:03:33.820 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: protected: False 2026-02-22T22:03:33.820 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T22:03:33.820 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T22:03:33.820 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: 319a0448-b0a9-483d-b56f-b265254e2f2e 2026-02-22T22:03:33.820 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: false 2026-02-22T22:03:33.821 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:03:33.821 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ compare_image_snaps mirror image_50 snap 2026-02-22T22:03:33.821 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:03:33.822 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_50 2026-02-22T22:03:33.822 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap_name=snap 2026-02-22T22:03:33.822 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local ret=0 2026-02-22T22:03:33.822 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local rmt_export=/tmp/tmp.j44LZPowal/cluster2-mirror-image_50.export 2026-02-22T22:03:33.822 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local loc_export=/tmp/tmp.j44LZPowal/cluster1-mirror-image_50.export 2026-02-22T22:03:33.822 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-image_50.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_50.export 2026-02-22T22:03:33.822 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 -p mirror export image_50@snap /tmp/tmp.j44LZPowal/cluster2-mirror-image_50.export 2026-02-22T22:03:33.934 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:03:33.942 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster1 -p mirror export image_50@snap /tmp/tmp.j44LZPowal/cluster1-mirror-image_50.export 2026-02-22T22:03:34.066 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:03:34.075 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ cmp /tmp/tmp.j44LZPowal/cluster2-mirror-image_50.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_50.export 2026-02-22T22:03:34.124 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -f /tmp/tmp.j44LZPowal/cluster2-mirror-image_50.export /tmp/tmp.j44LZPowal/cluster1-mirror-image_50.export 2026-02-22T22:03:34.159 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:03:34.159 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ testlog 'TEST: delete many images' 2026-02-22T22:03:34.159 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ tee -a /tmp/tmp.j44LZPowal/rbd-mirror.test.log 2026-02-22T22:03:34.159 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ date '+%F %T' 2026-02-22T22:03:34.160 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ echo 2026-02-22 22:03:34 TEST: delete many images 2026-02-22T22:03:34.160 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:2026-02-22 22:03:34 TEST: delete many images 2026-02-22T22:03:34.160 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ seq 1 50 2026-02-22T22:03:34.162 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:03:34.162 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_1 2026-02-22T22:03:34.162 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_snapshot cluster2 mirror image_1 snap 2026-02-22T22:03:34.162 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:03:34.162 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:03:34.162 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_1 2026-02-22T22:03:34.162 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap=snap 2026-02-22T22:03:34.162 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap rm mirror/image_1@snap 2026-02-22T22:03:34.240 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing snap: 100% complete...done. 2026-02-22T22:03:34.247 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image_retry cluster2 mirror image_1 2026-02-22T22:03:34.248 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:03:34.248 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:03:34.248 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_1 2026-02-22T22:03:34.248 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:03:34.248 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0 2026-02-22T22:03:34.249 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image cluster2 mirror image_1 2026-02-22T22:03:34.249 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:03:34.249 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:03:34.249 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_1 2026-02-22T22:03:34.249 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_1 2026-02-22T22:03:34.276 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 rm mirror/image_1 2026-02-22T22:03:34.335 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial026.stderr:2026-02-22T22:03:34.334+0000 7f5f98e81640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x56395249c340 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-22T22:03:34.335 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial026.stderr:2026-02-22T22:03:34.334+0000 7f5f9f68e640 -1 rbd::mirror::ImageReplayer: 0x563952b40a00 [3/90c63745-c08b-46eb-b4e8-44a4f1e91721] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-22T22:03:35.306 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T22:03:35.308 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:03:35.308 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:03:35.308 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_2 2026-02-22T22:03:35.308 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_snapshot cluster2 mirror image_2 snap 2026-02-22T22:03:35.308 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:03:35.308 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:03:35.308 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_2 2026-02-22T22:03:35.308 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap=snap 2026-02-22T22:03:35.308 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap rm mirror/image_2@snap 2026-02-22T22:03:35.382 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.29283500919107774) > weight (0.25) 2026-02-22T22:03:35.382 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.1 2026-02-22T22:03:35.383 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Sent signal 15 2026-02-22T22:03:35.383 INFO:tasks.rbd_mirror_thrash:waiting for 2.8356487605897494 secs before reviving daemons 2026-02-22T22:03:35.383 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial026.stderr:2026-02-22T22:03:35.382+0000 7f49bb912640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 (PID: 61853) UID: 1000 2026-02-22T22:03:36.246 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing snap: 100% complete...done. 2026-02-22T22:03:36.252 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image_retry cluster2 mirror image_2 2026-02-22T22:03:36.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:03:36.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:03:36.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_2 2026-02-22T22:03:36.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:03:36.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0 2026-02-22T22:03:36.254 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image cluster2 mirror image_2 2026-02-22T22:03:36.254 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:03:36.254 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:03:36.254 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_2 2026-02-22T22:03:36.254 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_2 2026-02-22T22:03:36.283 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 rm mirror/image_2 2026-02-22T22:03:36.338 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial026.stderr:2026-02-22T22:03:36.337+0000 7fdba1e13640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55ac3f74cb60 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-22T22:03:36.338 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial026.stderr:2026-02-22T22:03:36.337+0000 7fdba8620640 -1 rbd::mirror::ImageReplayer: 0x55ac3f838000 [3/9e277158-7fce-48dd-8563-7bdcdfac29d0] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-22T22:03:37.310 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T22:03:37.312 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:03:37.312 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:03:37.312 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_3 2026-02-22T22:03:37.312 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_snapshot cluster2 mirror image_3 snap 2026-02-22T22:03:37.313 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:03:37.313 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:03:37.313 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_3 2026-02-22T22:03:37.313 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap=snap 2026-02-22T22:03:37.313 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap rm mirror/image_3@snap 2026-02-22T22:03:38.219 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.1 2026-02-22T22:03:38.219 INFO:teuthology.orchestra.run:waiting for 600 2026-02-22T22:03:38.219 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.1 2026-02-22T22:03:38.219 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Restarting daemon 2026-02-22T22:03:38.219 DEBUG:teuthology.orchestra.run.trial026:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 2026-02-22T22:03:38.221 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Started 2026-02-22T22:03:38.221 INFO:tasks.rbd_mirror_thrash:waiting for 114.0246080875829 secs before thrashing 2026-02-22T22:03:38.250 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing snap: 100% complete...done. 2026-02-22T22:03:38.256 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image_retry cluster2 mirror image_3 2026-02-22T22:03:38.256 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:03:38.256 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:03:38.256 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_3 2026-02-22T22:03:38.256 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:03:38.256 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0 2026-02-22T22:03:38.256 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image cluster2 mirror image_3 2026-02-22T22:03:38.256 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:03:38.257 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:03:38.257 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_3 2026-02-22T22:03:38.257 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_3 2026-02-22T22:03:38.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 rm mirror/image_3 2026-02-22T22:03:38.342 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial026.stderr:2026-02-22T22:03:38.340+0000 7fdba1612640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55ac3f165ba0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-22T22:03:38.342 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial026.stderr:2026-02-22T22:03:38.340+0000 7fdba7e1f640 -1 rbd::mirror::ImageReplayer: 0x55ac3f7aa780 [3/06da309f-d597-43fe-8ebc-0e374ae46a50] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-22T22:03:39.308 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T22:03:39.310 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:03:39.310 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:03:39.310 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_4 2026-02-22T22:03:39.310 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_snapshot cluster2 mirror image_4 snap 2026-02-22T22:03:39.310 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:03:39.310 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:03:39.310 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_4 2026-02-22T22:03:39.310 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap=snap 2026-02-22T22:03:39.310 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap rm mirror/image_4@snap 2026-02-22T22:03:40.254 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing snap: 100% complete...done. 2026-02-22T22:03:40.260 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image_retry cluster2 mirror image_4 2026-02-22T22:03:40.260 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:03:40.261 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:03:40.261 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_4 2026-02-22T22:03:40.261 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:03:40.261 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0 2026-02-22T22:03:40.261 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image cluster2 mirror image_4 2026-02-22T22:03:40.262 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:03:40.262 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:03:40.262 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_4 2026-02-22T22:03:40.262 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_4 2026-02-22T22:03:40.290 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 rm mirror/image_4 2026-02-22T22:03:40.348 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial026.stderr:2026-02-22T22:03:40.347+0000 7fdba1612640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55ac3f74cea0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-22T22:03:40.348 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial026.stderr:2026-02-22T22:03:40.347+0000 7fdba8620640 -1 rbd::mirror::ImageReplayer: 0x55ac3f7aa000 [3/23c3bee2-bece-4830-b190-6b5f39e55cb1] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-22T22:03:41.317 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T22:03:41.319 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:03:41.319 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:03:41.319 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_5 2026-02-22T22:03:41.319 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_snapshot cluster2 mirror image_5 snap 2026-02-22T22:03:41.320 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:03:41.320 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:03:41.320 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_5 2026-02-22T22:03:41.320 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap=snap 2026-02-22T22:03:41.320 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap rm mirror/image_5@snap 2026-02-22T22:03:42.257 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing snap: 100% complete...done. 2026-02-22T22:03:42.263 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image_retry cluster2 mirror image_5 2026-02-22T22:03:42.263 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:03:42.263 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:03:42.263 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_5 2026-02-22T22:03:42.263 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:03:42.263 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0 2026-02-22T22:03:42.264 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image cluster2 mirror image_5 2026-02-22T22:03:42.264 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:03:42.264 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:03:42.264 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_5 2026-02-22T22:03:42.265 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_5 2026-02-22T22:03:42.293 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 rm mirror/image_5 2026-02-22T22:03:42.349 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial026.stderr:2026-02-22T22:03:42.348+0000 7f5f98e81640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x56395249c1a0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-22T22:03:42.350 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial026.stderr:2026-02-22T22:03:42.348+0000 7f5f9f68e640 -1 rbd::mirror::ImageReplayer: 0x563952b41680 [3/d7359255-a2a8-4833-b9d4-992b53784bf0] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-22T22:03:43.318 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T22:03:43.320 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:03:43.320 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:03:43.320 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_6 2026-02-22T22:03:43.320 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_snapshot cluster2 mirror image_6 snap 2026-02-22T22:03:43.320 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:03:43.320 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:03:43.320 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_6 2026-02-22T22:03:43.320 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap=snap 2026-02-22T22:03:43.320 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap rm mirror/image_6@snap 2026-02-22T22:03:44.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing snap: 100% complete...done. 2026-02-22T22:03:44.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image_retry cluster2 mirror image_6 2026-02-22T22:03:44.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:03:44.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:03:44.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_6 2026-02-22T22:03:44.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:03:44.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0 2026-02-22T22:03:44.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image cluster2 mirror image_6 2026-02-22T22:03:44.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:03:44.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:03:44.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_6 2026-02-22T22:03:44.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_6 2026-02-22T22:03:44.231 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 rm mirror/image_6 2026-02-22T22:03:44.290 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial026.stderr:2026-02-22T22:03:44.288+0000 7fdba1e13640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55ac3f74cd00 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-22T22:03:44.290 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial026.stderr:2026-02-22T22:03:44.288+0000 7fdba8620640 -1 rbd::mirror::ImageReplayer: 0x55ac3f7abb80 [3/763b8c73-d0f5-43b5-9453-e1f95dcf6809] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-22T22:03:45.322 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T22:03:45.324 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:03:45.324 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:03:45.324 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_7 2026-02-22T22:03:45.324 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_snapshot cluster2 mirror image_7 snap 2026-02-22T22:03:45.324 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:03:45.324 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:03:45.324 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_7 2026-02-22T22:03:45.324 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap=snap 2026-02-22T22:03:45.324 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap rm mirror/image_7@snap 2026-02-22T22:03:46.263 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing snap: 100% complete...done. 2026-02-22T22:03:46.268 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image_retry cluster2 mirror image_7 2026-02-22T22:03:46.268 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:03:46.268 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:03:46.268 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_7 2026-02-22T22:03:46.268 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:03:46.268 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0 2026-02-22T22:03:46.269 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image cluster2 mirror image_7 2026-02-22T22:03:46.269 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:03:46.269 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:03:46.269 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_7 2026-02-22T22:03:46.269 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_7 2026-02-22T22:03:46.297 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 rm mirror/image_7 2026-02-22T22:03:46.356 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial026.stderr:2026-02-22T22:03:46.355+0000 7fdba8620640 -1 rbd::mirror::ImageMap: 0x55ac3f495b00 handle_peer_ack_remove: failed to remove global_image_id=a383f702-7739-4131-97b5-30656d4ff503 2026-02-22T22:03:47.325 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T22:03:47.329 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:03:47.329 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:03:47.329 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_8 2026-02-22T22:03:47.329 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_snapshot cluster2 mirror image_8 snap 2026-02-22T22:03:47.329 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:03:47.329 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:03:47.330 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_8 2026-02-22T22:03:47.330 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap=snap 2026-02-22T22:03:47.330 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap rm mirror/image_8@snap 2026-02-22T22:03:48.268 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing snap: 100% complete...done. 2026-02-22T22:03:48.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image_retry cluster2 mirror image_8 2026-02-22T22:03:48.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:03:48.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:03:48.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_8 2026-02-22T22:03:48.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:03:48.273 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0 2026-02-22T22:03:48.273 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image cluster2 mirror image_8 2026-02-22T22:03:48.273 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:03:48.273 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:03:48.273 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_8 2026-02-22T22:03:48.273 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_8 2026-02-22T22:03:48.301 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 rm mirror/image_8 2026-02-22T22:03:48.362 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial026.stderr:2026-02-22T22:03:48.360+0000 7f5f98e81640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x563952aaf380 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-22T22:03:48.362 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial026.stderr:2026-02-22T22:03:48.360+0000 7f5f9f68e640 -1 rbd::mirror::ImageReplayer: 0x563952b40780 [3/6ec8d83c-acbe-4670-87cf-a72aa84767b5] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-22T22:03:49.251 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T22:03:49.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:03:49.254 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:03:49.254 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_9 2026-02-22T22:03:49.254 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_snapshot cluster2 mirror image_9 snap 2026-02-22T22:03:49.254 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:03:49.254 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:03:49.254 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_9 2026-02-22T22:03:49.254 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap=snap 2026-02-22T22:03:49.254 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap rm mirror/image_9@snap 2026-02-22T22:03:50.270 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing snap: 100% complete...done. 2026-02-22T22:03:50.276 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image_retry cluster2 mirror image_9 2026-02-22T22:03:50.276 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:03:50.276 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:03:50.276 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_9 2026-02-22T22:03:50.276 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:03:50.276 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0 2026-02-22T22:03:50.276 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image cluster2 mirror image_9 2026-02-22T22:03:50.276 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:03:50.276 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:03:50.277 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_9 2026-02-22T22:03:50.277 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_9 2026-02-22T22:03:50.306 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 rm mirror/image_9 2026-02-22T22:03:50.369 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial026.stderr:2026-02-22T22:03:50.368+0000 7f4b1f3b1640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x562edb0abd40 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-22T22:03:50.369 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial026.stderr:2026-02-22T22:03:50.368+0000 7f4b25bbe640 -1 rbd::mirror::ImageReplayer: 0x562edb768000 [3/cdba1f90-5b2a-43f5-aa74-a2544c65d250] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-22T22:03:51.336 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T22:03:51.339 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:03:51.339 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:03:51.339 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_10 2026-02-22T22:03:51.339 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_snapshot cluster2 mirror image_10 snap 2026-02-22T22:03:51.340 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:03:51.340 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:03:51.340 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_10 2026-02-22T22:03:51.340 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap=snap 2026-02-22T22:03:51.340 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap rm mirror/image_10@snap 2026-02-22T22:03:52.274 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing snap: 100% complete...done. 2026-02-22T22:03:52.279 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image_retry cluster2 mirror image_10 2026-02-22T22:03:52.279 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:03:52.279 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:03:52.279 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_10 2026-02-22T22:03:52.279 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:03:52.280 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0 2026-02-22T22:03:52.280 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image cluster2 mirror image_10 2026-02-22T22:03:52.280 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:03:52.281 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:03:52.281 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_10 2026-02-22T22:03:52.281 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_10 2026-02-22T22:03:52.310 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 rm mirror/image_10 2026-02-22T22:03:52.371 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial026.stderr:2026-02-22T22:03:52.369+0000 7efd7b9c1640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55d079b9e4e0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-22T22:03:52.371 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial026.stderr:2026-02-22T22:03:52.369+0000 7efd821ce640 -1 rbd::mirror::ImageReplayer: 0x55d0796ddb80 [3/0f526ee1-bae4-4f17-9197-c721da6ab0c9] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-22T22:03:53.335 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T22:03:53.337 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:03:53.338 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:03:53.338 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_11 2026-02-22T22:03:53.338 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_snapshot cluster2 mirror image_11 snap 2026-02-22T22:03:53.338 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:03:53.338 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:03:53.338 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_11 2026-02-22T22:03:53.338 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap=snap 2026-02-22T22:03:53.338 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap rm mirror/image_11@snap 2026-02-22T22:03:54.196 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing snap: 100% complete...done. 2026-02-22T22:03:54.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image_retry cluster2 mirror image_11 2026-02-22T22:03:54.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:03:54.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:03:54.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_11 2026-02-22T22:03:54.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:03:54.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0 2026-02-22T22:03:54.203 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image cluster2 mirror image_11 2026-02-22T22:03:54.203 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:03:54.203 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:03:54.203 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_11 2026-02-22T22:03:54.203 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_11 2026-02-22T22:03:54.231 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 rm mirror/image_11 2026-02-22T22:03:54.288 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial026.stderr:2026-02-22T22:03:54.287+0000 7fdba8620640 -1 rbd::mirror::ImageMap: 0x55ac3f495b00 handle_peer_ack_remove: failed to remove global_image_id=69c86d55-4653-4f0e-a6ff-bc5fe92ac722 2026-02-22T22:03:55.333 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T22:03:55.335 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:03:55.335 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:03:55.336 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_12 2026-02-22T22:03:55.336 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_snapshot cluster2 mirror image_12 snap 2026-02-22T22:03:55.336 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:03:55.336 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:03:55.336 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_12 2026-02-22T22:03:55.336 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap=snap 2026-02-22T22:03:55.336 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap rm mirror/image_12@snap 2026-02-22T22:03:56.283 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing snap: 100% complete...done. 2026-02-22T22:03:56.289 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image_retry cluster2 mirror image_12 2026-02-22T22:03:56.289 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:03:56.289 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:03:56.289 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_12 2026-02-22T22:03:56.289 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:03:56.289 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0 2026-02-22T22:03:56.290 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image cluster2 mirror image_12 2026-02-22T22:03:56.290 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:03:56.290 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:03:56.290 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_12 2026-02-22T22:03:56.290 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_12 2026-02-22T22:03:56.320 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 rm mirror/image_12 2026-02-22T22:03:56.375 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial026.stderr:2026-02-22T22:03:56.373+0000 7fdba7e1f640 -1 rbd::mirror::ImageMap: 0x55ac3f495b00 handle_peer_ack_remove: failed to remove global_image_id=82008379-765b-45b9-b3f2-d57783e88ea0 2026-02-22T22:03:57.341 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T22:03:57.344 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:03:57.345 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:03:57.345 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_13 2026-02-22T22:03:57.345 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_snapshot cluster2 mirror image_13 snap 2026-02-22T22:03:57.345 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:03:57.345 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:03:57.345 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_13 2026-02-22T22:03:57.345 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap=snap 2026-02-22T22:03:57.345 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap rm mirror/image_13@snap 2026-02-22T22:03:58.285 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing snap: 100% complete...done. 2026-02-22T22:03:58.290 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image_retry cluster2 mirror image_13 2026-02-22T22:03:58.290 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:03:58.290 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:03:58.291 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_13 2026-02-22T22:03:58.291 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:03:58.291 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0 2026-02-22T22:03:58.292 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image cluster2 mirror image_13 2026-02-22T22:03:58.292 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:03:58.292 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:03:58.292 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_13 2026-02-22T22:03:58.292 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_13 2026-02-22T22:03:58.320 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 rm mirror/image_13 2026-02-22T22:03:58.378 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial026.stderr:2026-02-22T22:03:58.375+0000 7fdba1612640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55ac3f16d040 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-22T22:03:58.378 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial026.stderr:2026-02-22T22:03:58.375+0000 7fdba8620640 -1 rbd::mirror::ImageReplayer: 0x55ac3f7aa280 [3/5d0b4c97-6ffd-4ecc-87a0-5cb43cf91b88] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-22T22:03:59.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T22:03:59.255 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:03:59.255 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:03:59.256 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_14 2026-02-22T22:03:59.256 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_snapshot cluster2 mirror image_14 snap 2026-02-22T22:03:59.256 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:03:59.256 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:03:59.256 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_14 2026-02-22T22:03:59.256 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap=snap 2026-02-22T22:03:59.256 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap rm mirror/image_14@snap 2026-02-22T22:04:00.289 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing snap: 100% complete...done. 2026-02-22T22:04:00.294 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image_retry cluster2 mirror image_14 2026-02-22T22:04:00.294 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:04:00.294 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:04:00.294 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_14 2026-02-22T22:04:00.294 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:04:00.294 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0 2026-02-22T22:04:00.295 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image cluster2 mirror image_14 2026-02-22T22:04:00.295 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:04:00.295 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:04:00.295 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_14 2026-02-22T22:04:00.295 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_14 2026-02-22T22:04:00.328 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 rm mirror/image_14 2026-02-22T22:04:00.385 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial026.stderr:2026-02-22T22:04:00.382+0000 7fdba7e1f640 -1 rbd::mirror::ImageMap: 0x55ac3f495b00 handle_peer_ack_remove: failed to remove global_image_id=adaf312a-e129-41dc-a0bd-6d745384fd91 2026-02-22T22:04:01.351 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T22:04:01.355 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:04:01.355 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:04:01.355 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_15 2026-02-22T22:04:01.355 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_snapshot cluster2 mirror image_15 snap 2026-02-22T22:04:01.355 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:04:01.355 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:04:01.355 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_15 2026-02-22T22:04:01.355 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap=snap 2026-02-22T22:04:01.355 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap rm mirror/image_15@snap 2026-02-22T22:04:02.290 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing snap: 100% complete...done. 2026-02-22T22:04:02.296 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image_retry cluster2 mirror image_15 2026-02-22T22:04:02.297 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:04:02.297 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:04:02.297 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_15 2026-02-22T22:04:02.297 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:04:02.297 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0 2026-02-22T22:04:02.298 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image cluster2 mirror image_15 2026-02-22T22:04:02.298 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:04:02.298 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:04:02.298 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_15 2026-02-22T22:04:02.298 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_15 2026-02-22T22:04:02.326 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 rm mirror/image_15 2026-02-22T22:04:03.358 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T22:04:03.361 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:04:03.361 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:04:03.361 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_16 2026-02-22T22:04:03.362 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_snapshot cluster2 mirror image_16 snap 2026-02-22T22:04:03.362 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:04:03.362 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:04:03.362 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_16 2026-02-22T22:04:03.362 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap=snap 2026-02-22T22:04:03.362 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap rm mirror/image_16@snap 2026-02-22T22:04:04.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing snap: 100% complete...done. 2026-02-22T22:04:04.203 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image_retry cluster2 mirror image_16 2026-02-22T22:04:04.203 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:04:04.203 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:04:04.203 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_16 2026-02-22T22:04:04.203 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:04:04.203 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0 2026-02-22T22:04:04.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image cluster2 mirror image_16 2026-02-22T22:04:04.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:04:04.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:04:04.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_16 2026-02-22T22:04:04.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_16 2026-02-22T22:04:04.234 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 rm mirror/image_16 2026-02-22T22:04:04.293 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial026.stderr:2026-02-22T22:04:04.290+0000 7efd7b9c1640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55d0795a4340 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-22T22:04:04.293 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial026.stderr:2026-02-22T22:04:04.290+0000 7efd821ce640 -1 rbd::mirror::ImageReplayer: 0x55d079bdc000 [3/16696fcf-2c1b-4676-bcbc-e837ecbffb01] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-22T22:04:05.257 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T22:04:05.261 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:04:05.261 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:04:05.261 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_17 2026-02-22T22:04:05.261 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_snapshot cluster2 mirror image_17 snap 2026-02-22T22:04:05.261 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:04:05.261 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:04:05.261 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_17 2026-02-22T22:04:05.261 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap=snap 2026-02-22T22:04:05.261 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap rm mirror/image_17@snap 2026-02-22T22:04:06.296 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing snap: 100% complete...done. 2026-02-22T22:04:06.302 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image_retry cluster2 mirror image_17 2026-02-22T22:04:06.302 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:04:06.302 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:04:06.302 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_17 2026-02-22T22:04:06.302 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:04:06.302 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0 2026-02-22T22:04:06.303 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image cluster2 mirror image_17 2026-02-22T22:04:06.303 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:04:06.303 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:04:06.303 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_17 2026-02-22T22:04:06.303 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_17 2026-02-22T22:04:06.332 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 rm mirror/image_17 2026-02-22T22:04:06.391 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial026.stderr:2026-02-22T22:04:06.389+0000 7f4b1f3b1640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x562edb0abba0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-22T22:04:06.391 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial026.stderr:2026-02-22T22:04:06.389+0000 7f4b25bbe640 -1 rbd::mirror::ImageReplayer: 0x562edb251b80 [3/0c4cedeb-aa9b-42fc-8183-74c2a5e4fd57] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-22T22:04:07.360 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T22:04:07.363 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:04:07.363 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:04:07.363 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_18 2026-02-22T22:04:07.363 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_snapshot cluster2 mirror image_18 snap 2026-02-22T22:04:07.363 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:04:07.363 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:04:07.364 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_18 2026-02-22T22:04:07.364 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap=snap 2026-02-22T22:04:07.364 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap rm mirror/image_18@snap 2026-02-22T22:04:08.302 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing snap: 100% complete...done. 2026-02-22T22:04:08.308 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image_retry cluster2 mirror image_18 2026-02-22T22:04:08.308 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:04:08.308 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:04:08.308 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_18 2026-02-22T22:04:08.309 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:04:08.309 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0 2026-02-22T22:04:08.310 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image cluster2 mirror image_18 2026-02-22T22:04:08.310 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:04:08.310 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:04:08.310 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_18 2026-02-22T22:04:08.310 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_18 2026-02-22T22:04:08.338 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 rm mirror/image_18 2026-02-22T22:04:08.394 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial026.stderr:2026-02-22T22:04:08.393+0000 7fdba1612640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55ac3f74cd00 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-22T22:04:08.394 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial026.stderr:2026-02-22T22:04:08.393+0000 7fdba8620640 -1 rbd::mirror::ImageReplayer: 0x55ac3f7aaf00 [3/4baf6590-1204-4386-a935-9b243b94dcfc] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-22T22:04:09.264 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T22:04:09.266 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:04:09.266 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:04:09.266 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_19 2026-02-22T22:04:09.267 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_snapshot cluster2 mirror image_19 snap 2026-02-22T22:04:09.267 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:04:09.267 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:04:09.267 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_19 2026-02-22T22:04:09.267 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap=snap 2026-02-22T22:04:09.267 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap rm mirror/image_19@snap 2026-02-22T22:04:10.301 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing snap: 100% complete...done. 2026-02-22T22:04:10.306 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image_retry cluster2 mirror image_19 2026-02-22T22:04:10.306 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:04:10.306 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:04:10.306 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_19 2026-02-22T22:04:10.306 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:04:10.306 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0 2026-02-22T22:04:10.307 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image cluster2 mirror image_19 2026-02-22T22:04:10.307 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:04:10.307 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:04:10.307 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_19 2026-02-22T22:04:10.307 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_19 2026-02-22T22:04:10.340 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 rm mirror/image_19 2026-02-22T22:04:10.396 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial026.stderr:2026-02-22T22:04:10.394+0000 7efd7b9c1640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55d079b9eb60 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-22T22:04:10.396 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial026.stderr:2026-02-22T22:04:10.394+0000 7efd821ce640 -1 rbd::mirror::ImageReplayer: 0x55d079bdc000 [3/7a33a362-8595-45cc-9e4c-7b4215120e99] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-22T22:04:11.362 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T22:04:11.364 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:04:11.364 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:04:11.365 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_20 2026-02-22T22:04:11.365 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_snapshot cluster2 mirror image_20 snap 2026-02-22T22:04:11.365 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:04:11.365 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:04:11.365 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_20 2026-02-22T22:04:11.365 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap=snap 2026-02-22T22:04:11.365 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap rm mirror/image_20@snap 2026-02-22T22:04:12.308 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing snap: 100% complete...done. 2026-02-22T22:04:12.313 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image_retry cluster2 mirror image_20 2026-02-22T22:04:12.313 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:04:12.313 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:04:12.313 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_20 2026-02-22T22:04:12.313 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:04:12.313 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0 2026-02-22T22:04:12.314 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image cluster2 mirror image_20 2026-02-22T22:04:12.314 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:04:12.314 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:04:12.314 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_20 2026-02-22T22:04:12.314 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_20 2026-02-22T22:04:12.343 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 rm mirror/image_20 2026-02-22T22:04:12.400 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial026.stderr:2026-02-22T22:04:12.398+0000 7fdba1e13640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55ac3f74cea0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-22T22:04:12.400 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial026.stderr:2026-02-22T22:04:12.398+0000 7fdba8620640 -1 rbd::mirror::ImageReplayer: 0x55ac3f7ab180 [3/50d0516c-a4f2-4b0b-9b37-15ee9b73d3ad] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-22T22:04:13.372 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T22:04:13.374 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:04:13.375 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:04:13.375 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_21 2026-02-22T22:04:13.375 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_snapshot cluster2 mirror image_21 snap 2026-02-22T22:04:13.375 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:04:13.375 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:04:13.375 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_21 2026-02-22T22:04:13.375 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap=snap 2026-02-22T22:04:13.375 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap rm mirror/image_21@snap 2026-02-22T22:04:14.199 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing snap: 100% complete...done. 2026-02-22T22:04:14.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image_retry cluster2 mirror image_21 2026-02-22T22:04:14.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:04:14.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:04:14.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_21 2026-02-22T22:04:14.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:04:14.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0 2026-02-22T22:04:14.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image cluster2 mirror image_21 2026-02-22T22:04:14.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:04:14.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:04:14.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_21 2026-02-22T22:04:14.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_21 2026-02-22T22:04:14.235 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 rm mirror/image_21 2026-02-22T22:04:14.290 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial026.stderr:2026-02-22T22:04:14.289+0000 7efd7b1c0640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55d07a432340 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-22T22:04:14.291 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial026.stderr:2026-02-22T22:04:14.289+0000 7efd821ce640 -1 rbd::mirror::ImageReplayer: 0x55d079bdd680 [3/fcfc28ec-1e7e-4846-a752-e36745d49c74] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-22T22:04:15.258 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T22:04:15.261 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:04:15.261 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:04:15.261 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_22 2026-02-22T22:04:15.261 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_snapshot cluster2 mirror image_22 snap 2026-02-22T22:04:15.262 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:04:15.262 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:04:15.262 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_22 2026-02-22T22:04:15.262 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap=snap 2026-02-22T22:04:15.262 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap rm mirror/image_22@snap 2026-02-22T22:04:16.198 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing snap: 100% complete...done. 2026-02-22T22:04:16.203 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image_retry cluster2 mirror image_22 2026-02-22T22:04:16.203 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:04:16.203 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:04:16.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_22 2026-02-22T22:04:16.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:04:16.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0 2026-02-22T22:04:16.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image cluster2 mirror image_22 2026-02-22T22:04:16.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:04:16.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:04:16.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_22 2026-02-22T22:04:16.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_22 2026-02-22T22:04:16.234 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 rm mirror/image_22 2026-02-22T22:04:16.291 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial026.stderr:2026-02-22T22:04:16.289+0000 7fdba1612640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55ac3f4db380 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-22T22:04:16.291 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial026.stderr:2026-02-22T22:04:16.289+0000 7fdba7e1f640 -1 rbd::mirror::ImageReplayer: 0x55ac3f838500 [3/cbed6911-da61-47b0-a3cc-2ddeb6616c4c] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-22T22:04:17.258 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T22:04:17.261 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:04:17.261 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:04:17.261 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_23 2026-02-22T22:04:17.262 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_snapshot cluster2 mirror image_23 snap 2026-02-22T22:04:17.262 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:04:17.262 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:04:17.262 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_23 2026-02-22T22:04:17.262 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap=snap 2026-02-22T22:04:17.262 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap rm mirror/image_23@snap 2026-02-22T22:04:17.514 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.9132503157635057) > weight (0.25) 2026-02-22T22:04:17.515 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.1 2026-02-22T22:04:17.515 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Sent signal 15 2026-02-22T22:04:17.515 INFO:tasks.rbd_mirror_thrash:waiting for 3.505318231876573 secs before reviving daemons 2026-02-22T22:04:17.516 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial026.stderr:2026-02-22T22:04:17.515+0000 7f8b686b5640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 (PID: 73346) UID: 1000 2026-02-22T22:04:18.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing snap: 100% complete...done. 2026-02-22T22:04:18.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image_retry cluster2 mirror image_23 2026-02-22T22:04:18.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:04:18.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:04:18.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_23 2026-02-22T22:04:18.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:04:18.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0 2026-02-22T22:04:18.213 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image cluster2 mirror image_23 2026-02-22T22:04:18.213 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:04:18.213 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:04:18.213 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_23 2026-02-22T22:04:18.213 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_23 2026-02-22T22:04:18.246 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 rm mirror/image_23 2026-02-22T22:04:18.306 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial026.stderr:2026-02-22T22:04:18.304+0000 7efd7b9c1640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55d0795a4340 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-22T22:04:18.306 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial026.stderr:2026-02-22T22:04:18.304+0000 7efd821ce640 -1 rbd::mirror::ImageReplayer: 0x55d079bdd400 [3/707a358f-f62b-47ff-a4e3-ae016127a25e] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-22T22:04:19.255 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T22:04:19.257 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:04:19.257 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:04:19.257 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_24 2026-02-22T22:04:19.257 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_snapshot cluster2 mirror image_24 snap 2026-02-22T22:04:19.258 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:04:19.258 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:04:19.258 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_24 2026-02-22T22:04:19.258 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap=snap 2026-02-22T22:04:19.258 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap rm mirror/image_24@snap 2026-02-22T22:04:20.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing snap: 100% complete...done. 2026-02-22T22:04:20.214 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image_retry cluster2 mirror image_24 2026-02-22T22:04:20.215 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:04:20.215 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:04:20.215 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_24 2026-02-22T22:04:20.215 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:04:20.215 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0 2026-02-22T22:04:20.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image cluster2 mirror image_24 2026-02-22T22:04:20.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:04:20.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:04:20.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_24 2026-02-22T22:04:20.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_24 2026-02-22T22:04:20.245 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 rm mirror/image_24 2026-02-22T22:04:20.306 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial026.stderr:2026-02-22T22:04:20.305+0000 7f4b1f3b1640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x562edb6b3040 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-22T22:04:20.306 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial026.stderr:2026-02-22T22:04:20.305+0000 7f4b25bbe640 -1 rbd::mirror::ImageReplayer: 0x562edb768280 [3/f06205bf-4d40-464a-9efe-4169410d86ad] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-22T22:04:21.022 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.1 2026-02-22T22:04:21.022 INFO:teuthology.orchestra.run:waiting for 600 2026-02-22T22:04:21.022 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.1 2026-02-22T22:04:21.022 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Restarting daemon 2026-02-22T22:04:21.022 DEBUG:teuthology.orchestra.run.trial026:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 2026-02-22T22:04:21.024 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Started 2026-02-22T22:04:21.025 INFO:tasks.rbd_mirror_thrash:waiting for 64.29568437820629 secs before thrashing 2026-02-22T22:04:21.267 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T22:04:21.270 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:04:21.270 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:04:21.270 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_25 2026-02-22T22:04:21.270 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_snapshot cluster2 mirror image_25 snap 2026-02-22T22:04:21.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:04:21.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:04:21.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_25 2026-02-22T22:04:21.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap=snap 2026-02-22T22:04:21.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap rm mirror/image_25@snap 2026-02-22T22:04:22.214 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing snap: 100% complete...done. 2026-02-22T22:04:22.219 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image_retry cluster2 mirror image_25 2026-02-22T22:04:22.219 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:04:22.220 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:04:22.220 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_25 2026-02-22T22:04:22.220 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:04:22.220 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0 2026-02-22T22:04:22.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image cluster2 mirror image_25 2026-02-22T22:04:22.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:04:22.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:04:22.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_25 2026-02-22T22:04:22.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_25 2026-02-22T22:04:22.254 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 rm mirror/image_25 2026-02-22T22:04:22.310 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial026.stderr:2026-02-22T22:04:22.309+0000 7f5f98680640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x56395249c340 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-22T22:04:22.311 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial026.stderr:2026-02-22T22:04:22.309+0000 7f5f9ee8d640 -1 rbd::mirror::ImageReplayer: 0x563952b40c80 [3/886eb68e-ef31-471e-9872-4fcf569230c4] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-22T22:04:23.282 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T22:04:23.285 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:04:23.285 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:04:23.286 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_26 2026-02-22T22:04:23.286 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_snapshot cluster2 mirror image_26 snap 2026-02-22T22:04:23.286 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:04:23.286 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:04:23.286 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_26 2026-02-22T22:04:23.286 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap=snap 2026-02-22T22:04:23.286 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap rm mirror/image_26@snap 2026-02-22T22:04:24.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing snap: 100% complete...done. 2026-02-22T22:04:24.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image_retry cluster2 mirror image_26 2026-02-22T22:04:24.209 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:04:24.209 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:04:24.209 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_26 2026-02-22T22:04:24.209 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:04:24.209 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0 2026-02-22T22:04:24.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image cluster2 mirror image_26 2026-02-22T22:04:24.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:04:24.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:04:24.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_26 2026-02-22T22:04:24.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_26 2026-02-22T22:04:24.238 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 rm mirror/image_26 2026-02-22T22:04:24.294 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial026.stderr:2026-02-22T22:04:24.293+0000 7fdba1e13640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55ac3f16ab60 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-22T22:04:24.294 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial026.stderr:2026-02-22T22:04:24.293+0000 7fdba7e1f640 -1 rbd::mirror::ImageReplayer: 0x55ac3f838a00 [3/f2b9fbe9-fa15-4b10-aead-5e1775df224b] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-22T22:04:25.281 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T22:04:25.285 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:04:25.285 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:04:25.285 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_27 2026-02-22T22:04:25.285 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_snapshot cluster2 mirror image_27 snap 2026-02-22T22:04:25.285 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:04:25.285 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:04:25.285 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_27 2026-02-22T22:04:25.285 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap=snap 2026-02-22T22:04:25.285 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap rm mirror/image_27@snap 2026-02-22T22:04:26.222 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing snap: 100% complete...done. 2026-02-22T22:04:26.228 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image_retry cluster2 mirror image_27 2026-02-22T22:04:26.228 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:04:26.228 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:04:26.228 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_27 2026-02-22T22:04:26.228 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:04:26.228 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0 2026-02-22T22:04:26.229 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image cluster2 mirror image_27 2026-02-22T22:04:26.229 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:04:26.229 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:04:26.229 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_27 2026-02-22T22:04:26.229 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_27 2026-02-22T22:04:26.256 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 rm mirror/image_27 2026-02-22T22:04:26.311 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial026.stderr:2026-02-22T22:04:26.309+0000 7f5f98e81640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x56395249c820 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-22T22:04:26.311 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial026.stderr:2026-02-22T22:04:26.309+0000 7f5f9ee8d640 -1 rbd::mirror::ImageReplayer: 0x563952607400 [3/226f7166-427f-4ef9-893a-6179cde62b6d] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-22T22:04:27.281 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T22:04:27.283 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:04:27.283 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:04:27.283 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_28 2026-02-22T22:04:27.283 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_snapshot cluster2 mirror image_28 snap 2026-02-22T22:04:27.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:04:27.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:04:27.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_28 2026-02-22T22:04:27.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap=snap 2026-02-22T22:04:27.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap rm mirror/image_28@snap 2026-02-22T22:04:28.228 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing snap: 100% complete...done. 2026-02-22T22:04:28.233 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image_retry cluster2 mirror image_28 2026-02-22T22:04:28.233 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:04:28.233 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:04:28.233 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_28 2026-02-22T22:04:28.233 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:04:28.233 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0 2026-02-22T22:04:28.233 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image cluster2 mirror image_28 2026-02-22T22:04:28.233 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:04:28.233 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:04:28.233 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_28 2026-02-22T22:04:28.234 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_28 2026-02-22T22:04:28.262 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 rm mirror/image_28 2026-02-22T22:04:28.318 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial026.stderr:2026-02-22T22:04:28.317+0000 7f4b1f3b1640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x562edb6b3d40 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-22T22:04:28.318 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial026.stderr:2026-02-22T22:04:28.317+0000 7f4b25bbe640 -1 rbd::mirror::ImageReplayer: 0x562edb6e1180 [3/8b05d225-eedc-4b0a-94f7-6d3c0ca41d35] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-22T22:04:29.262 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T22:04:29.266 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:04:29.266 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:04:29.266 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_29 2026-02-22T22:04:29.266 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_snapshot cluster2 mirror image_29 snap 2026-02-22T22:04:29.266 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:04:29.266 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:04:29.266 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_29 2026-02-22T22:04:29.266 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap=snap 2026-02-22T22:04:29.266 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap rm mirror/image_29@snap 2026-02-22T22:04:30.229 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing snap: 100% complete...done. 2026-02-22T22:04:30.234 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image_retry cluster2 mirror image_29 2026-02-22T22:04:30.234 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:04:30.234 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:04:30.234 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_29 2026-02-22T22:04:30.234 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:04:30.234 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0 2026-02-22T22:04:30.235 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image cluster2 mirror image_29 2026-02-22T22:04:30.235 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:04:30.235 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:04:30.235 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_29 2026-02-22T22:04:30.235 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_29 2026-02-22T22:04:30.264 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 rm mirror/image_29 2026-02-22T22:04:30.325 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial026.stderr:2026-02-22T22:04:30.324+0000 7f4b1f3b1640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x562edb6b2820 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-22T22:04:30.325 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial026.stderr:2026-02-22T22:04:30.324+0000 7f4b25bbe640 -1 rbd::mirror::ImageReplayer: 0x562edb6e0280 [3/199d7709-f322-4604-895e-a583a38cb8d5] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-22T22:04:31.295 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T22:04:31.297 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:04:31.298 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:04:31.298 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_30 2026-02-22T22:04:31.298 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_snapshot cluster2 mirror image_30 snap 2026-02-22T22:04:31.298 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:04:31.298 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:04:31.298 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_30 2026-02-22T22:04:31.298 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap=snap 2026-02-22T22:04:31.298 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap rm mirror/image_30@snap 2026-02-22T22:04:32.233 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing snap: 100% complete...done. 2026-02-22T22:04:32.239 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image_retry cluster2 mirror image_30 2026-02-22T22:04:32.239 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:04:32.240 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:04:32.240 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_30 2026-02-22T22:04:32.240 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:04:32.240 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0 2026-02-22T22:04:32.241 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image cluster2 mirror image_30 2026-02-22T22:04:32.241 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:04:32.241 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:04:32.241 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_30 2026-02-22T22:04:32.241 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_30 2026-02-22T22:04:32.269 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 rm mirror/image_30 2026-02-22T22:04:32.328 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial026.stderr:2026-02-22T22:04:32.325+0000 7efd7b1c0640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55d07a2fd520 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-22T22:04:32.328 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial026.stderr:2026-02-22T22:04:32.325+0000 7efd821ce640 -1 rbd::mirror::ImageReplayer: 0x55d079bdd180 [3/cce0d5f1-d366-4b28-ba1a-4d3fe01fd2e1] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-22T22:04:33.293 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T22:04:33.297 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:04:33.298 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:04:33.298 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_31 2026-02-22T22:04:33.298 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_snapshot cluster2 mirror image_31 snap 2026-02-22T22:04:33.298 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:04:33.298 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:04:33.298 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_31 2026-02-22T22:04:33.298 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap=snap 2026-02-22T22:04:33.298 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap rm mirror/image_31@snap 2026-02-22T22:04:34.201 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing snap: 100% complete...done. 2026-02-22T22:04:34.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image_retry cluster2 mirror image_31 2026-02-22T22:04:34.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:04:34.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:04:34.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_31 2026-02-22T22:04:34.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:04:34.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0 2026-02-22T22:04:34.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image cluster2 mirror image_31 2026-02-22T22:04:34.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:04:34.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:04:34.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_31 2026-02-22T22:04:34.208 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_31 2026-02-22T22:04:34.236 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 rm mirror/image_31 2026-02-22T22:04:34.295 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial026.stderr:2026-02-22T22:04:34.293+0000 7efd7b1c0640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55d07a2fd380 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-22T22:04:34.295 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial026.stderr:2026-02-22T22:04:34.293+0000 7efd821ce640 -1 rbd::mirror::ImageReplayer: 0x55d079bdcc80 [3/97a4414e-20e7-4288-9c80-0dce42f41d4f] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-22T22:04:35.294 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T22:04:35.298 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:04:35.298 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:04:35.298 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_32 2026-02-22T22:04:35.298 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_snapshot cluster2 mirror image_32 snap 2026-02-22T22:04:35.298 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:04:35.298 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:04:35.298 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_32 2026-02-22T22:04:35.298 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap=snap 2026-02-22T22:04:35.298 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap rm mirror/image_32@snap 2026-02-22T22:04:36.241 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing snap: 100% complete...done. 2026-02-22T22:04:36.246 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image_retry cluster2 mirror image_32 2026-02-22T22:04:36.246 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:04:36.246 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:04:36.246 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_32 2026-02-22T22:04:36.246 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:04:36.246 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0 2026-02-22T22:04:36.247 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image cluster2 mirror image_32 2026-02-22T22:04:36.247 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:04:36.247 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:04:36.248 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_32 2026-02-22T22:04:36.248 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_32 2026-02-22T22:04:36.276 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 rm mirror/image_32 2026-02-22T22:04:36.331 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial026.stderr:2026-02-22T22:04:36.329+0000 7f4b1f3b1640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x562edb429860 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-22T22:04:36.332 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial026.stderr:2026-02-22T22:04:36.329+0000 7f4b263bf640 -1 rbd::mirror::ImageReplayer: 0x562edb6e0a00 [3/533da63c-6a64-4a63-a224-d042b1c3553d] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-22T22:04:37.303 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T22:04:37.307 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:04:37.307 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:04:37.307 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_33 2026-02-22T22:04:37.307 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_snapshot cluster2 mirror image_33 snap 2026-02-22T22:04:37.307 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:04:37.307 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:04:37.308 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_33 2026-02-22T22:04:37.308 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap=snap 2026-02-22T22:04:37.308 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap rm mirror/image_33@snap 2026-02-22T22:04:38.246 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing snap: 100% complete...done. 2026-02-22T22:04:38.251 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image_retry cluster2 mirror image_33 2026-02-22T22:04:38.252 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:04:38.252 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:04:38.252 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_33 2026-02-22T22:04:38.252 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:04:38.252 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0 2026-02-22T22:04:38.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image cluster2 mirror image_33 2026-02-22T22:04:38.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:04:38.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:04:38.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_33 2026-02-22T22:04:38.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_33 2026-02-22T22:04:38.282 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 rm mirror/image_33 2026-02-22T22:04:38.341 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial026.stderr:2026-02-22T22:04:38.339+0000 7fdba1612640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55ac3f16a9c0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-22T22:04:38.342 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial026.stderr:2026-02-22T22:04:38.339+0000 7fdba7e1f640 -1 rbd::mirror::ImageReplayer: 0x55ac3f838000 [3/ad9fe32c-2e40-4486-ab81-178c7a9c3a27] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-22T22:04:39.268 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T22:04:39.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:04:39.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:04:39.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_34 2026-02-22T22:04:39.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_snapshot cluster2 mirror image_34 snap 2026-02-22T22:04:39.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:04:39.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:04:39.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_34 2026-02-22T22:04:39.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap=snap 2026-02-22T22:04:39.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap rm mirror/image_34@snap 2026-02-22T22:04:40.249 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing snap: 100% complete...done. 2026-02-22T22:04:40.254 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image_retry cluster2 mirror image_34 2026-02-22T22:04:40.255 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:04:40.255 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:04:40.255 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_34 2026-02-22T22:04:40.255 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:04:40.255 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0 2026-02-22T22:04:40.256 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image cluster2 mirror image_34 2026-02-22T22:04:40.256 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:04:40.256 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:04:40.256 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_34 2026-02-22T22:04:40.256 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_34 2026-02-22T22:04:40.289 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 rm mirror/image_34 2026-02-22T22:04:40.328 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial026.stderr:2026-02-22T22:04:40.326+0000 7f1e6f569640 -1 rbd::mirror::ImageMap: 0x55fab60e6000 handle_peer_ack_remove: failed to remove global_image_id=787c83b7-1192-4f94-aa31-23be72fbd343 2026-02-22T22:04:41.311 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T22:04:41.313 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:04:41.313 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:04:41.313 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_35 2026-02-22T22:04:41.314 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_snapshot cluster2 mirror image_35 snap 2026-02-22T22:04:41.314 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:04:41.314 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:04:41.314 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_35 2026-02-22T22:04:41.314 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap=snap 2026-02-22T22:04:41.314 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap rm mirror/image_35@snap 2026-02-22T22:04:42.255 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing snap: 100% complete...done. 2026-02-22T22:04:42.261 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image_retry cluster2 mirror image_35 2026-02-22T22:04:42.261 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:04:42.261 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:04:42.261 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_35 2026-02-22T22:04:42.262 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:04:42.262 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0 2026-02-22T22:04:42.262 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image cluster2 mirror image_35 2026-02-22T22:04:42.263 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:04:42.263 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:04:42.263 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_35 2026-02-22T22:04:42.263 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_35 2026-02-22T22:04:42.291 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 rm mirror/image_35 2026-02-22T22:04:42.355 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial026.stderr:2026-02-22T22:04:42.353+0000 7fdba1612640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55ac3f74da00 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-22T22:04:42.355 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial026.stderr:2026-02-22T22:04:42.353+0000 7fdba8620640 -1 rbd::mirror::ImageReplayer: 0x55ac3f7ab400 [3/7d8cf4a3-1fa1-4dcc-9d4d-2c1a44d0099a] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-22T22:04:43.317 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T22:04:43.321 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:04:43.321 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:04:43.321 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_36 2026-02-22T22:04:43.321 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_snapshot cluster2 mirror image_36 snap 2026-02-22T22:04:43.322 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:04:43.322 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:04:43.322 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_36 2026-02-22T22:04:43.322 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap=snap 2026-02-22T22:04:43.322 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap rm mirror/image_36@snap 2026-02-22T22:04:44.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing snap: 100% complete...done. 2026-02-22T22:04:44.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image_retry cluster2 mirror image_36 2026-02-22T22:04:44.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:04:44.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:04:44.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_36 2026-02-22T22:04:44.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:04:44.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0 2026-02-22T22:04:44.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image cluster2 mirror image_36 2026-02-22T22:04:44.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:04:44.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:04:44.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_36 2026-02-22T22:04:44.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_36 2026-02-22T22:04:44.241 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 rm mirror/image_36 2026-02-22T22:04:44.299 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial026.stderr:2026-02-22T22:04:44.297+0000 7f4b1fbb2640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x562edb429520 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-22T22:04:44.299 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial026.stderr:2026-02-22T22:04:44.297+0000 7f4b263bf640 -1 rbd::mirror::ImageReplayer: 0x562edb6e1400 [3/9860c66a-05af-466f-baca-3772f5e211ad] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-22T22:04:45.317 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T22:04:45.319 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:04:45.319 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:04:45.319 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_37 2026-02-22T22:04:45.319 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_snapshot cluster2 mirror image_37 snap 2026-02-22T22:04:45.319 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:04:45.319 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:04:45.319 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_37 2026-02-22T22:04:45.319 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap=snap 2026-02-22T22:04:45.319 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap rm mirror/image_37@snap 2026-02-22T22:04:46.263 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing snap: 100% complete...done. 2026-02-22T22:04:46.270 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image_retry cluster2 mirror image_37 2026-02-22T22:04:46.270 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:04:46.270 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:04:46.270 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_37 2026-02-22T22:04:46.270 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:04:46.270 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0 2026-02-22T22:04:46.270 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image cluster2 mirror image_37 2026-02-22T22:04:46.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:04:46.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:04:46.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_37 2026-02-22T22:04:46.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_37 2026-02-22T22:04:46.300 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 rm mirror/image_37 2026-02-22T22:04:46.356 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial026.stderr:2026-02-22T22:04:46.354+0000 7f4b1fbb2640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x562edb6b36c0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-22T22:04:46.356 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial026.stderr:2026-02-22T22:04:46.354+0000 7f4b263bf640 -1 rbd::mirror::ImageReplayer: 0x562edb6e0000 [3/0ecb5a5c-9fb3-4659-bd3e-6eb17ad9c7a3] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-22T22:04:47.330 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T22:04:47.333 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:04:47.333 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:04:47.333 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_38 2026-02-22T22:04:47.333 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_snapshot cluster2 mirror image_38 snap 2026-02-22T22:04:47.333 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:04:47.333 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:04:47.333 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_38 2026-02-22T22:04:47.333 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap=snap 2026-02-22T22:04:47.333 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap rm mirror/image_38@snap 2026-02-22T22:04:48.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing snap: 100% complete...done. 2026-02-22T22:04:48.277 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image_retry cluster2 mirror image_38 2026-02-22T22:04:48.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:04:48.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:04:48.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_38 2026-02-22T22:04:48.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:04:48.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0 2026-02-22T22:04:48.279 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image cluster2 mirror image_38 2026-02-22T22:04:48.280 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:04:48.280 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:04:48.280 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_38 2026-02-22T22:04:48.280 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_38 2026-02-22T22:04:48.308 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 rm mirror/image_38 2026-02-22T22:04:48.367 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial026.stderr:2026-02-22T22:04:48.365+0000 7fdba1e13640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55ac3f74c9c0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-22T22:04:48.367 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial026.stderr:2026-02-22T22:04:48.365+0000 7fdba8620640 -1 rbd::mirror::ImageReplayer: 0x55ac3f2f5b80 [3/db20ecda-1e60-4b4e-b471-d368349418c1] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-22T22:04:49.267 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T22:04:49.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:04:49.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:04:49.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_39 2026-02-22T22:04:49.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_snapshot cluster2 mirror image_39 snap 2026-02-22T22:04:49.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:04:49.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:04:49.273 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_39 2026-02-22T22:04:49.273 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap=snap 2026-02-22T22:04:49.273 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap rm mirror/image_39@snap 2026-02-22T22:04:50.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing snap: 100% complete...done. 2026-02-22T22:04:50.277 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image_retry cluster2 mirror image_39 2026-02-22T22:04:50.277 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:04:50.277 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:04:50.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_39 2026-02-22T22:04:50.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:04:50.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0 2026-02-22T22:04:50.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image cluster2 mirror image_39 2026-02-22T22:04:50.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:04:50.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:04:50.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_39 2026-02-22T22:04:50.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_39 2026-02-22T22:04:50.307 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 rm mirror/image_39 2026-02-22T22:04:50.362 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial026.stderr:2026-02-22T22:04:50.361+0000 7f4b1fbb2640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x562edb6b3520 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-22T22:04:50.363 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial026.stderr:2026-02-22T22:04:50.361+0000 7f4b263bf640 -1 rbd::mirror::ImageReplayer: 0x562edb6e0c80 [3/92432590-5c9b-4a0c-b269-4ed3db4a7657] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-22T22:04:51.334 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T22:04:51.337 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:04:51.337 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:04:51.337 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_40 2026-02-22T22:04:51.337 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_snapshot cluster2 mirror image_40 snap 2026-02-22T22:04:51.337 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:04:51.337 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:04:51.337 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_40 2026-02-22T22:04:51.337 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap=snap 2026-02-22T22:04:51.337 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap rm mirror/image_40@snap 2026-02-22T22:04:52.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing snap: 100% complete...done. 2026-02-22T22:04:52.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image_retry cluster2 mirror image_40 2026-02-22T22:04:52.285 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:04:52.285 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:04:52.285 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_40 2026-02-22T22:04:52.285 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:04:52.285 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0 2026-02-22T22:04:52.286 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image cluster2 mirror image_40 2026-02-22T22:04:52.286 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:04:52.286 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:04:52.286 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_40 2026-02-22T22:04:52.286 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_40 2026-02-22T22:04:52.319 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 rm mirror/image_40 2026-02-22T22:04:52.376 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial026.stderr:2026-02-22T22:04:52.374+0000 7f4b1f3b1640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x562edb428d00 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-22T22:04:52.376 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial026.stderr:2026-02-22T22:04:52.374+0000 7f4b263bf640 -1 rbd::mirror::ImageReplayer: 0x562edb6e1680 [3/c57ae3a7-f4fd-4810-a324-ac5f588ec54e] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-22T22:04:53.343 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T22:04:53.345 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:04:53.345 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:04:53.345 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_41 2026-02-22T22:04:53.345 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_snapshot cluster2 mirror image_41 snap 2026-02-22T22:04:53.345 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:04:53.345 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:04:53.345 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_41 2026-02-22T22:04:53.345 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap=snap 2026-02-22T22:04:53.345 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap rm mirror/image_41@snap 2026-02-22T22:04:54.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing snap: 100% complete...done. 2026-02-22T22:04:54.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image_retry cluster2 mirror image_41 2026-02-22T22:04:54.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:04:54.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:04:54.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_41 2026-02-22T22:04:54.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:04:54.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0 2026-02-22T22:04:54.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image cluster2 mirror image_41 2026-02-22T22:04:54.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:04:54.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:04:54.213 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_41 2026-02-22T22:04:54.213 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_41 2026-02-22T22:04:54.243 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 rm mirror/image_41 2026-02-22T22:04:54.301 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial026.stderr:2026-02-22T22:04:54.300+0000 7f5f98680640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x563952aae1a0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-22T22:04:54.302 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial026.stderr:2026-02-22T22:04:54.300+0000 7f5f9f68e640 -1 rbd::mirror::ImageReplayer: 0x563952b40000 [3/69209f84-531b-4558-bff7-497f660589ca] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-22T22:04:55.342 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T22:04:55.344 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:04:55.344 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:04:55.344 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_42 2026-02-22T22:04:55.345 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_snapshot cluster2 mirror image_42 snap 2026-02-22T22:04:55.345 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:04:55.345 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:04:55.345 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_42 2026-02-22T22:04:55.345 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap=snap 2026-02-22T22:04:55.345 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap rm mirror/image_42@snap 2026-02-22T22:04:56.287 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing snap: 100% complete...done. 2026-02-22T22:04:56.293 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image_retry cluster2 mirror image_42 2026-02-22T22:04:56.293 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:04:56.293 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:04:56.293 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_42 2026-02-22T22:04:56.293 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:04:56.294 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0 2026-02-22T22:04:56.295 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image cluster2 mirror image_42 2026-02-22T22:04:56.295 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:04:56.295 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:04:56.295 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_42 2026-02-22T22:04:56.295 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_42 2026-02-22T22:04:56.324 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 rm mirror/image_42 2026-02-22T22:04:56.382 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial026.stderr:2026-02-22T22:04:56.380+0000 7f5f98e81640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x563952aaf860 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-22T22:04:56.382 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial026.stderr:2026-02-22T22:04:56.380+0000 7f5f9f68e640 -1 rbd::mirror::ImageReplayer: 0x563952b40500 [3/75d1530f-c13b-4f15-aac1-c10cd70d3738] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-22T22:04:57.347 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T22:04:57.351 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:04:57.351 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:04:57.351 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_43 2026-02-22T22:04:57.351 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_snapshot cluster2 mirror image_43 snap 2026-02-22T22:04:57.351 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:04:57.352 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:04:57.352 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_43 2026-02-22T22:04:57.352 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap=snap 2026-02-22T22:04:57.352 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap rm mirror/image_43@snap 2026-02-22T22:04:58.290 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing snap: 100% complete...done. 2026-02-22T22:04:58.297 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image_retry cluster2 mirror image_43 2026-02-22T22:04:58.297 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:04:58.297 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:04:58.297 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_43 2026-02-22T22:04:58.297 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:04:58.298 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0 2026-02-22T22:04:58.299 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image cluster2 mirror image_43 2026-02-22T22:04:58.299 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:04:58.299 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:04:58.299 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_43 2026-02-22T22:04:58.299 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_43 2026-02-22T22:04:58.327 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 rm mirror/image_43 2026-02-22T22:04:58.384 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial026.stderr:2026-02-22T22:04:58.383+0000 7f5f98e81640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x563952aae9c0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-22T22:04:58.385 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial026.stderr:2026-02-22T22:04:58.383+0000 7f5f9f68e640 -1 rbd::mirror::ImageReplayer: 0x563952b40f00 [3/af18f3fc-e37e-4800-a4fb-c06589d02d67] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-22T22:04:59.264 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T22:04:59.266 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:04:59.266 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:04:59.266 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_44 2026-02-22T22:04:59.266 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_snapshot cluster2 mirror image_44 snap 2026-02-22T22:04:59.266 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:04:59.266 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:04:59.266 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_44 2026-02-22T22:04:59.266 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap=snap 2026-02-22T22:04:59.266 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap rm mirror/image_44@snap 2026-02-22T22:05:00.291 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing snap: 100% complete...done. 2026-02-22T22:05:00.297 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image_retry cluster2 mirror image_44 2026-02-22T22:05:00.297 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:05:00.297 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:00.297 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_44 2026-02-22T22:05:00.297 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:05:00.297 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0 2026-02-22T22:05:00.298 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image cluster2 mirror image_44 2026-02-22T22:05:00.298 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:05:00.298 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:00.298 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_44 2026-02-22T22:05:00.298 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_44 2026-02-22T22:05:00.327 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 rm mirror/image_44 2026-02-22T22:05:00.385 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial026.stderr:2026-02-22T22:05:00.383+0000 7f5f98680640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x563952aae680 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-22T22:05:01.357 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T22:05:01.359 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:05:01.360 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:05:01.360 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_45 2026-02-22T22:05:01.360 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_snapshot cluster2 mirror image_45 snap 2026-02-22T22:05:01.360 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:05:01.360 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:01.360 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_45 2026-02-22T22:05:01.360 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap=snap 2026-02-22T22:05:01.360 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap rm mirror/image_45@snap 2026-02-22T22:05:02.300 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing snap: 100% complete...done. 2026-02-22T22:05:02.305 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image_retry cluster2 mirror image_45 2026-02-22T22:05:02.305 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:05:02.305 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:02.305 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_45 2026-02-22T22:05:02.306 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:05:02.306 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0 2026-02-22T22:05:02.306 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image cluster2 mirror image_45 2026-02-22T22:05:02.306 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:05:02.306 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:02.306 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_45 2026-02-22T22:05:02.306 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_45 2026-02-22T22:05:02.335 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 rm mirror/image_45 2026-02-22T22:05:02.394 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial026.stderr:2026-02-22T22:05:02.392+0000 7f5f98680640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x563952aae4e0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-22T22:05:02.394 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial026.stderr:2026-02-22T22:05:02.392+0000 7f5f9f68e640 -1 rbd::mirror::ImageReplayer: 0x563952607180 [3/1024109e-7bb5-4ed3-ab78-ad539d98c9fd] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-22T22:05:03.362 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T22:05:03.365 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:05:03.365 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:05:03.365 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_46 2026-02-22T22:05:03.365 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_snapshot cluster2 mirror image_46 snap 2026-02-22T22:05:03.365 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:05:03.365 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:03.365 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_46 2026-02-22T22:05:03.365 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap=snap 2026-02-22T22:05:03.366 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap rm mirror/image_46@snap 2026-02-22T22:05:04.207 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing snap: 100% complete...done. 2026-02-22T22:05:04.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image_retry cluster2 mirror image_46 2026-02-22T22:05:04.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:05:04.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:04.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_46 2026-02-22T22:05:04.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:05:04.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0 2026-02-22T22:05:04.213 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image cluster2 mirror image_46 2026-02-22T22:05:04.213 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:05:04.213 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:04.213 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_46 2026-02-22T22:05:04.213 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_46 2026-02-22T22:05:04.242 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 rm mirror/image_46 2026-02-22T22:05:04.304 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial026.stderr:2026-02-22T22:05:04.303+0000 7f4b1f3b1640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x562edb0b3d40 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-22T22:05:04.305 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial026.stderr:2026-02-22T22:05:04.303+0000 7f4b25bbe640 -1 rbd::mirror::ImageReplayer: 0x562edb6e0780 [3/04845371-74eb-4487-a2bf-eec2351b191c] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-22T22:05:05.264 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T22:05:05.266 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:05:05.266 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:05:05.266 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_47 2026-02-22T22:05:05.266 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_snapshot cluster2 mirror image_47 snap 2026-02-22T22:05:05.266 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:05:05.266 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:05.266 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_47 2026-02-22T22:05:05.266 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap=snap 2026-02-22T22:05:05.266 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap rm mirror/image_47@snap 2026-02-22T22:05:06.304 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing snap: 100% complete...done. 2026-02-22T22:05:06.310 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image_retry cluster2 mirror image_47 2026-02-22T22:05:06.310 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:05:06.310 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:06.310 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_47 2026-02-22T22:05:06.310 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:05:06.310 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0 2026-02-22T22:05:06.311 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image cluster2 mirror image_47 2026-02-22T22:05:06.311 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:05:06.311 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:06.312 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_47 2026-02-22T22:05:06.312 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_47 2026-02-22T22:05:06.344 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 rm mirror/image_47 2026-02-22T22:05:06.402 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial026.stderr:2026-02-22T22:05:06.400+0000 7fdba1612640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55ac3f74d860 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-22T22:05:06.402 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial026.stderr:2026-02-22T22:05:06.400+0000 7fdba8620640 -1 rbd::mirror::ImageReplayer: 0x55ac3f7aac80 [3/1c0dd8a5-46f9-44d8-9de4-48e37b2678f7] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-22T22:05:07.356 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T22:05:07.358 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:05:07.358 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:05:07.358 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_48 2026-02-22T22:05:07.358 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_snapshot cluster2 mirror image_48 snap 2026-02-22T22:05:07.358 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:05:07.358 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:07.358 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_48 2026-02-22T22:05:07.359 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap=snap 2026-02-22T22:05:07.359 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap rm mirror/image_48@snap 2026-02-22T22:05:08.304 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing snap: 100% complete...done. 2026-02-22T22:05:08.309 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image_retry cluster2 mirror image_48 2026-02-22T22:05:08.309 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:05:08.309 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:08.309 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_48 2026-02-22T22:05:08.309 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:05:08.309 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0 2026-02-22T22:05:08.310 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image cluster2 mirror image_48 2026-02-22T22:05:08.310 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:05:08.310 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:08.310 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_48 2026-02-22T22:05:08.310 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_48 2026-02-22T22:05:08.338 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 rm mirror/image_48 2026-02-22T22:05:08.397 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial026.stderr:2026-02-22T22:05:08.395+0000 7f5f98e81640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x563952aaf520 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-22T22:05:08.397 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial026.stderr:2026-02-22T22:05:08.395+0000 7f5f9f68e640 -1 rbd::mirror::ImageReplayer: 0x563952b41180 [3/073c2a09-55c1-4f22-ad52-735988b4ed7b] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-22T22:05:09.269 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T22:05:09.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:05:09.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:05:09.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_49 2026-02-22T22:05:09.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_snapshot cluster2 mirror image_49 snap 2026-02-22T22:05:09.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:05:09.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:09.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_49 2026-02-22T22:05:09.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap=snap 2026-02-22T22:05:09.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap rm mirror/image_49@snap 2026-02-22T22:05:10.308 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing snap: 100% complete...done. 2026-02-22T22:05:10.315 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image_retry cluster2 mirror image_49 2026-02-22T22:05:10.315 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:05:10.315 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:10.315 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_49 2026-02-22T22:05:10.315 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:05:10.315 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0 2026-02-22T22:05:10.316 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image cluster2 mirror image_49 2026-02-22T22:05:10.316 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:05:10.316 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:10.317 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_49 2026-02-22T22:05:10.317 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_49 2026-02-22T22:05:10.346 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 rm mirror/image_49 2026-02-22T22:05:10.404 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial026.stderr:2026-02-22T22:05:10.402+0000 7fdba1612640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55ac400ddd40 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-22T22:05:10.404 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial026.stderr:2026-02-22T22:05:10.402+0000 7fdba7e1f640 -1 rbd::mirror::ImageReplayer: 0x55ac3f7ab900 [3/61a34d68-de2f-48ac-99bc-627b57103d64] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-22T22:05:11.373 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T22:05:11.376 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:05:11.376 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:05:11.376 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_50 2026-02-22T22:05:11.376 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_snapshot cluster2 mirror image_50 snap 2026-02-22T22:05:11.376 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:05:11.376 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:11.377 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_50 2026-02-22T22:05:11.377 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local snap=snap 2026-02-22T22:05:11.377 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap rm mirror/image_50@snap 2026-02-22T22:05:12.311 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing snap: 100% complete...done. 2026-02-22T22:05:12.318 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image_retry cluster2 mirror image_50 2026-02-22T22:05:12.318 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:05:12.319 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:12.319 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_50 2026-02-22T22:05:12.319 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:05:12.319 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0 2026-02-22T22:05:12.320 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image cluster2 mirror image_50 2026-02-22T22:05:12.320 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:05:12.320 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:12.320 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_50 2026-02-22T22:05:12.320 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_50 2026-02-22T22:05:12.348 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 rm mirror/image_50 2026-02-22T22:05:12.406 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial026.stderr:2026-02-22T22:05:12.403+0000 7f5f98680640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x563952aaed00 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-22T22:05:12.406 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial026.stderr:2026-02-22T22:05:12.403+0000 7f5f9f68e640 -1 rbd::mirror::ImageReplayer: 0x563952607680 [3/319a0448-b0a9-483d-b56f-b265254e2f2e] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-22T22:05:13.572 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T22:05:13.574 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:05:13.574 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ testlog 'TEST: image deletions should propagate' 2026-02-22T22:05:13.574 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ tee -a /tmp/tmp.j44LZPowal/rbd-mirror.test.log 2026-02-22T22:05:13.575 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ date '+%F %T' 2026-02-22T22:05:13.576 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ echo 2026-02-22 22:05:13 TEST: image deletions should propagate 2026-02-22T22:05:13.576 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:2026-02-22 22:05:13 TEST: image deletions should propagate 2026-02-22T22:05:13.576 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_pool_images cluster1 mirror 0 2026-02-22T22:05:13.576 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:13.576 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:13.576 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_count=0 2026-02-22T22:05:13.576 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:05:13.576 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local count 2026-02-22T22:05:13.576 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local last_count=0 2026-02-22T22:05:13.576 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ true 2026-02-22T22:05:13.576 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ seq 1 40 2026-02-22T22:05:13.577 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in `seq 1 40` 2026-02-22T22:05:13.577 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 1 -ne 1 2026-02-22T22:05:13.577 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster cluster1 -p mirror mirror pool status 2026-02-22T22:05:13.578 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ grep 'images: ' 2026-02-22T22:05:13.578 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cut '-d ' -f 2 2026-02-22T22:05:13.601 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: failed to query services: (13) Permission denied 2026-02-22T22:05:13.602 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ count=0 2026-02-22T22:05:13.602 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 0 = 0 2026-02-22T22:05:13.602 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:05:13.602 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_pool_healthy cluster1 mirror 0 2026-02-22T22:05:13.602 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:13.602 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:13.602 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:05:13.602 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state 2026-02-22T22:05:13.602 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ seq 1 40 2026-02-22T22:05:13.603 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in `seq 1 40` 2026-02-22T22:05:13.603 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 1 -ne 1 2026-02-22T22:05:13.603 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster cluster1 -p mirror mirror pool status 2026-02-22T22:05:13.604 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ grep 'image health:' 2026-02-22T22:05:13.604 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cut '-d ' -f 3 2026-02-22T22:05:13.626 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: failed to query services: (13) Permission denied 2026-02-22T22:05:13.628 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ state=OK 2026-02-22T22:05:13.628 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test OK = ERROR 2026-02-22T22:05:13.628 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test OK = OK 2026-02-22T22:05:13.628 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:05:13.628 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ seq 1 50 2026-02-22T22:05:13.629 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:05:13.629 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_1 2026-02-22T22:05:13.629 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_present cluster1 mirror image_1 deleted 2026-02-22T22:05:13.629 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:13.629 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:13.629 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_1 2026-02-22T22:05:13.629 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=deleted 2026-02-22T22:05:13.629 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:05:13.629 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:05:13.629 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:05:13.629 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_1 2026-02-22T22:05:13.629 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:05:13.629 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:05:13.629 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_1 2026-02-22T22:05:13.630 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_1 2026-02-22T22:05:13.630 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:05:13.651 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_1: (2) No such file or directory 2026-02-22T22:05:13.653 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image_id= 2026-02-22T22:05:13.653 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:05:13.653 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.1 2026-02-22T22:05:13.754 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_present cluster1 mirror image_1 deleted '' 2026-02-22T22:05:13.754 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:13.754 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:13.754 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_1 2026-02-22T22:05:13.754 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=deleted 2026-02-22T22:05:13.754 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:05:13.755 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=deleted 2026-02-22T22:05:13.755 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_image_id 2026-02-22T22:05:13.755 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_1 2026-02-22T22:05:13.755 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:05:13.755 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:05:13.755 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_1 2026-02-22T22:05:13.755 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_1 2026-02-22T22:05:13.755 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:05:13.776 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_1: (2) No such file or directory 2026-02-22T22:05:13.778 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_image_id= 2026-02-22T22:05:13.778 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:05:13.778 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test deleted = deleted 2026-02-22T22:05:13.778 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:05:13.778 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:05:13.778 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_2 2026-02-22T22:05:13.779 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_present cluster1 mirror image_2 deleted 2026-02-22T22:05:13.779 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:13.779 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:13.779 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_2 2026-02-22T22:05:13.779 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=deleted 2026-02-22T22:05:13.779 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:05:13.779 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:05:13.779 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:05:13.779 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_2 2026-02-22T22:05:13.779 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:05:13.779 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:05:13.779 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_2 2026-02-22T22:05:13.779 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_2 2026-02-22T22:05:13.779 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:05:13.801 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_2: (2) No such file or directory 2026-02-22T22:05:13.803 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image_id= 2026-02-22T22:05:13.804 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:05:13.804 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.1 2026-02-22T22:05:13.904 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_present cluster1 mirror image_2 deleted '' 2026-02-22T22:05:13.905 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:13.905 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:13.905 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_2 2026-02-22T22:05:13.905 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=deleted 2026-02-22T22:05:13.905 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:05:13.905 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=deleted 2026-02-22T22:05:13.905 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_image_id 2026-02-22T22:05:13.905 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_2 2026-02-22T22:05:13.905 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:05:13.905 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:05:13.905 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_2 2026-02-22T22:05:13.906 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_2 2026-02-22T22:05:13.906 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:05:13.926 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_2: (2) No such file or directory 2026-02-22T22:05:13.929 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_image_id= 2026-02-22T22:05:13.929 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:05:13.930 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test deleted = deleted 2026-02-22T22:05:13.930 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:05:13.930 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:05:13.930 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_3 2026-02-22T22:05:13.930 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_present cluster1 mirror image_3 deleted 2026-02-22T22:05:13.930 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:13.930 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:13.930 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_3 2026-02-22T22:05:13.930 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=deleted 2026-02-22T22:05:13.930 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:05:13.930 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:05:13.931 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:05:13.931 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_3 2026-02-22T22:05:13.931 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:05:13.931 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:05:13.931 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_3 2026-02-22T22:05:13.931 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_3 2026-02-22T22:05:13.931 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:05:13.952 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_3: (2) No such file or directory 2026-02-22T22:05:13.953 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image_id= 2026-02-22T22:05:13.954 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:05:13.954 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.1 2026-02-22T22:05:14.055 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_present cluster1 mirror image_3 deleted '' 2026-02-22T22:05:14.055 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:14.055 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:14.055 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_3 2026-02-22T22:05:14.056 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=deleted 2026-02-22T22:05:14.056 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:05:14.056 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=deleted 2026-02-22T22:05:14.056 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_image_id 2026-02-22T22:05:14.056 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_3 2026-02-22T22:05:14.056 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:05:14.056 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:05:14.056 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_3 2026-02-22T22:05:14.056 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_3 2026-02-22T22:05:14.056 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:05:14.077 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_3: (2) No such file or directory 2026-02-22T22:05:14.079 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_image_id= 2026-02-22T22:05:14.079 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:05:14.079 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test deleted = deleted 2026-02-22T22:05:14.079 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:05:14.079 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:05:14.079 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_4 2026-02-22T22:05:14.079 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_present cluster1 mirror image_4 deleted 2026-02-22T22:05:14.079 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:14.079 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:14.079 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_4 2026-02-22T22:05:14.079 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=deleted 2026-02-22T22:05:14.079 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:05:14.079 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:05:14.079 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:05:14.079 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_4 2026-02-22T22:05:14.079 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:05:14.079 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:05:14.079 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_4 2026-02-22T22:05:14.080 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_4 2026-02-22T22:05:14.080 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:05:14.101 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_4: (2) No such file or directory 2026-02-22T22:05:14.103 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image_id= 2026-02-22T22:05:14.103 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:05:14.103 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.1 2026-02-22T22:05:14.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_present cluster1 mirror image_4 deleted '' 2026-02-22T22:05:14.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:14.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:14.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_4 2026-02-22T22:05:14.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=deleted 2026-02-22T22:05:14.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:05:14.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=deleted 2026-02-22T22:05:14.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_image_id 2026-02-22T22:05:14.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_4 2026-02-22T22:05:14.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:05:14.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:05:14.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_4 2026-02-22T22:05:14.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_4 2026-02-22T22:05:14.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:05:14.228 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_4: (2) No such file or directory 2026-02-22T22:05:14.231 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_image_id= 2026-02-22T22:05:14.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:05:14.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test deleted = deleted 2026-02-22T22:05:14.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:05:14.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:05:14.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_5 2026-02-22T22:05:14.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_present cluster1 mirror image_5 deleted 2026-02-22T22:05:14.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:14.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:14.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_5 2026-02-22T22:05:14.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=deleted 2026-02-22T22:05:14.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:05:14.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:05:14.233 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:05:14.233 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_5 2026-02-22T22:05:14.233 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:05:14.233 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:05:14.233 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_5 2026-02-22T22:05:14.233 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_5 2026-02-22T22:05:14.233 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:05:14.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_5: (2) No such file or directory 2026-02-22T22:05:14.255 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image_id= 2026-02-22T22:05:14.255 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:05:14.255 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.1 2026-02-22T22:05:14.357 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_present cluster1 mirror image_5 deleted '' 2026-02-22T22:05:14.358 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:14.358 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:14.358 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_5 2026-02-22T22:05:14.358 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=deleted 2026-02-22T22:05:14.358 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:05:14.358 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=deleted 2026-02-22T22:05:14.358 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_image_id 2026-02-22T22:05:14.358 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_5 2026-02-22T22:05:14.358 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:05:14.359 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:05:14.359 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_5 2026-02-22T22:05:14.359 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_5 2026-02-22T22:05:14.359 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:05:14.378 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_5: (2) No such file or directory 2026-02-22T22:05:14.381 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_image_id= 2026-02-22T22:05:14.381 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:05:14.382 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test deleted = deleted 2026-02-22T22:05:14.382 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:05:14.382 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:05:14.382 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_6 2026-02-22T22:05:14.382 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_present cluster1 mirror image_6 deleted 2026-02-22T22:05:14.382 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:14.382 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:14.382 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_6 2026-02-22T22:05:14.382 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=deleted 2026-02-22T22:05:14.382 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:05:14.382 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:05:14.383 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:05:14.383 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_6 2026-02-22T22:05:14.383 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:05:14.383 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:05:14.383 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_6 2026-02-22T22:05:14.383 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_6 2026-02-22T22:05:14.383 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:05:14.402 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_6: (2) No such file or directory 2026-02-22T22:05:14.404 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image_id= 2026-02-22T22:05:14.404 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:05:14.404 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.1 2026-02-22T22:05:14.507 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_present cluster1 mirror image_6 deleted '' 2026-02-22T22:05:14.508 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:14.508 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:14.508 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_6 2026-02-22T22:05:14.508 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=deleted 2026-02-22T22:05:14.508 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:05:14.508 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=deleted 2026-02-22T22:05:14.509 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_image_id 2026-02-22T22:05:14.509 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_6 2026-02-22T22:05:14.509 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:05:14.509 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:05:14.509 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_6 2026-02-22T22:05:14.509 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_6 2026-02-22T22:05:14.510 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:05:14.529 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_6: (2) No such file or directory 2026-02-22T22:05:14.532 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_image_id= 2026-02-22T22:05:14.532 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:05:14.532 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test deleted = deleted 2026-02-22T22:05:14.532 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:05:14.533 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:05:14.533 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_7 2026-02-22T22:05:14.533 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_present cluster1 mirror image_7 deleted 2026-02-22T22:05:14.533 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:14.533 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:14.533 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_7 2026-02-22T22:05:14.533 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=deleted 2026-02-22T22:05:14.533 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:05:14.533 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:05:14.533 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:05:14.533 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_7 2026-02-22T22:05:14.533 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:05:14.534 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:05:14.534 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_7 2026-02-22T22:05:14.534 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_7 2026-02-22T22:05:14.534 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:05:14.554 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_7: (2) No such file or directory 2026-02-22T22:05:14.556 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image_id= 2026-02-22T22:05:14.557 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:05:14.557 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.1 2026-02-22T22:05:14.659 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_present cluster1 mirror image_7 deleted '' 2026-02-22T22:05:14.660 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:14.660 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:14.660 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_7 2026-02-22T22:05:14.660 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=deleted 2026-02-22T22:05:14.660 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:05:14.661 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=deleted 2026-02-22T22:05:14.661 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_image_id 2026-02-22T22:05:14.661 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_7 2026-02-22T22:05:14.661 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:05:14.661 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:05:14.661 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_7 2026-02-22T22:05:14.661 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_7 2026-02-22T22:05:14.662 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:05:14.682 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_7: (2) No such file or directory 2026-02-22T22:05:14.684 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_image_id= 2026-02-22T22:05:14.685 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:05:14.685 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test deleted = deleted 2026-02-22T22:05:14.685 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:05:14.685 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:05:14.685 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_8 2026-02-22T22:05:14.685 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_present cluster1 mirror image_8 deleted 2026-02-22T22:05:14.685 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:14.685 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:14.685 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_8 2026-02-22T22:05:14.685 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=deleted 2026-02-22T22:05:14.685 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:05:14.685 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:05:14.685 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:05:14.686 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_8 2026-02-22T22:05:14.686 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:05:14.686 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:05:14.686 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_8 2026-02-22T22:05:14.686 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_8 2026-02-22T22:05:14.686 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:05:14.707 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_8: (2) No such file or directory 2026-02-22T22:05:14.708 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image_id= 2026-02-22T22:05:14.708 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:05:14.708 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.1 2026-02-22T22:05:14.809 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_present cluster1 mirror image_8 deleted '' 2026-02-22T22:05:14.809 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:14.809 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:14.809 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_8 2026-02-22T22:05:14.809 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=deleted 2026-02-22T22:05:14.809 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:05:14.809 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=deleted 2026-02-22T22:05:14.809 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_image_id 2026-02-22T22:05:14.810 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_8 2026-02-22T22:05:14.810 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:05:14.810 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:05:14.810 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_8 2026-02-22T22:05:14.810 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_8 2026-02-22T22:05:14.810 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:05:14.832 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_8: (2) No such file or directory 2026-02-22T22:05:14.834 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_image_id= 2026-02-22T22:05:14.834 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:05:14.834 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test deleted = deleted 2026-02-22T22:05:14.834 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:05:14.834 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:05:14.834 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_9 2026-02-22T22:05:14.834 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_present cluster1 mirror image_9 deleted 2026-02-22T22:05:14.834 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:14.834 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:14.834 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_9 2026-02-22T22:05:14.834 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=deleted 2026-02-22T22:05:14.834 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:05:14.834 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:05:14.834 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:05:14.834 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_9 2026-02-22T22:05:14.834 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:05:14.834 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:05:14.834 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_9 2026-02-22T22:05:14.835 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_9 2026-02-22T22:05:14.835 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:05:14.857 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_9: (2) No such file or directory 2026-02-22T22:05:14.858 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image_id= 2026-02-22T22:05:14.858 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:05:14.859 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.1 2026-02-22T22:05:14.962 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_present cluster1 mirror image_9 deleted '' 2026-02-22T22:05:14.962 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:14.962 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:14.963 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_9 2026-02-22T22:05:14.963 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=deleted 2026-02-22T22:05:14.963 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:05:14.963 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=deleted 2026-02-22T22:05:14.963 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_image_id 2026-02-22T22:05:14.963 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_9 2026-02-22T22:05:14.963 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:05:14.964 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:05:14.964 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_9 2026-02-22T22:05:14.964 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_9 2026-02-22T22:05:14.964 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:05:14.985 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_9: (2) No such file or directory 2026-02-22T22:05:14.988 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_image_id= 2026-02-22T22:05:14.988 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:05:14.988 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test deleted = deleted 2026-02-22T22:05:14.988 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:05:14.988 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:05:14.988 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_10 2026-02-22T22:05:14.988 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_present cluster1 mirror image_10 deleted 2026-02-22T22:05:14.988 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:14.988 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:14.988 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_10 2026-02-22T22:05:14.989 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=deleted 2026-02-22T22:05:14.989 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:05:14.989 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:05:14.989 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:05:14.989 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_10 2026-02-22T22:05:14.989 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:05:14.989 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:05:14.989 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_10 2026-02-22T22:05:14.989 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_10 2026-02-22T22:05:14.989 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:05:15.009 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_10: (2) No such file or directory 2026-02-22T22:05:15.011 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image_id= 2026-02-22T22:05:15.011 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:05:15.011 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.1 2026-02-22T22:05:15.112 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_present cluster1 mirror image_10 deleted '' 2026-02-22T22:05:15.112 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:15.112 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:15.113 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_10 2026-02-22T22:05:15.113 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=deleted 2026-02-22T22:05:15.113 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:05:15.113 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=deleted 2026-02-22T22:05:15.113 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_image_id 2026-02-22T22:05:15.113 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_10 2026-02-22T22:05:15.113 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:05:15.113 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:05:15.113 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_10 2026-02-22T22:05:15.113 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_10 2026-02-22T22:05:15.113 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:05:15.135 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_10: (2) No such file or directory 2026-02-22T22:05:15.137 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_image_id= 2026-02-22T22:05:15.137 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:05:15.137 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test deleted = deleted 2026-02-22T22:05:15.137 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:05:15.137 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:05:15.137 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_11 2026-02-22T22:05:15.137 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_present cluster1 mirror image_11 deleted 2026-02-22T22:05:15.137 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:15.137 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:15.137 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_11 2026-02-22T22:05:15.137 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=deleted 2026-02-22T22:05:15.137 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:05:15.137 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:05:15.137 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:05:15.137 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_11 2026-02-22T22:05:15.137 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:05:15.137 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:05:15.137 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_11 2026-02-22T22:05:15.137 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_11 2026-02-22T22:05:15.138 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:05:15.361 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_11: (2) No such file or directory 2026-02-22T22:05:15.362 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image_id= 2026-02-22T22:05:15.362 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:05:15.362 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.1 2026-02-22T22:05:15.464 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_present cluster1 mirror image_11 deleted '' 2026-02-22T22:05:15.465 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:15.465 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:15.465 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_11 2026-02-22T22:05:15.465 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=deleted 2026-02-22T22:05:15.465 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:05:15.465 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=deleted 2026-02-22T22:05:15.465 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_image_id 2026-02-22T22:05:15.465 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_11 2026-02-22T22:05:15.465 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:05:15.465 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:05:15.465 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_11 2026-02-22T22:05:15.465 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_11 2026-02-22T22:05:15.465 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:05:15.488 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_11: (2) No such file or directory 2026-02-22T22:05:15.491 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_image_id= 2026-02-22T22:05:15.491 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:05:15.491 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test deleted = deleted 2026-02-22T22:05:15.491 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:05:15.491 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:05:15.491 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_12 2026-02-22T22:05:15.491 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_present cluster1 mirror image_12 deleted 2026-02-22T22:05:15.491 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:15.491 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:15.491 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_12 2026-02-22T22:05:15.491 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=deleted 2026-02-22T22:05:15.491 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:05:15.491 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:05:15.491 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:05:15.491 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_12 2026-02-22T22:05:15.491 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:05:15.491 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:05:15.491 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_12 2026-02-22T22:05:15.491 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_12 2026-02-22T22:05:15.492 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:05:15.513 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_12: (2) No such file or directory 2026-02-22T22:05:15.515 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image_id= 2026-02-22T22:05:15.515 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:05:15.515 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.1 2026-02-22T22:05:15.616 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_present cluster1 mirror image_12 deleted '' 2026-02-22T22:05:15.617 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:15.617 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:15.617 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_12 2026-02-22T22:05:15.617 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=deleted 2026-02-22T22:05:15.617 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:05:15.617 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=deleted 2026-02-22T22:05:15.617 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_image_id 2026-02-22T22:05:15.617 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_12 2026-02-22T22:05:15.617 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:05:15.617 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:05:15.617 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_12 2026-02-22T22:05:15.617 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_12 2026-02-22T22:05:15.617 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:05:15.641 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_12: (2) No such file or directory 2026-02-22T22:05:15.643 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_image_id= 2026-02-22T22:05:15.643 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:05:15.643 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test deleted = deleted 2026-02-22T22:05:15.643 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:05:15.643 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:05:15.643 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_13 2026-02-22T22:05:15.644 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_present cluster1 mirror image_13 deleted 2026-02-22T22:05:15.644 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:15.644 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:15.644 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_13 2026-02-22T22:05:15.644 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=deleted 2026-02-22T22:05:15.644 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:05:15.644 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:05:15.644 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:05:15.644 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_13 2026-02-22T22:05:15.644 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:05:15.644 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:05:15.644 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_13 2026-02-22T22:05:15.644 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_13 2026-02-22T22:05:15.644 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:05:15.867 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_13: (2) No such file or directory 2026-02-22T22:05:15.869 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image_id= 2026-02-22T22:05:15.869 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:05:15.869 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.1 2026-02-22T22:05:15.971 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_present cluster1 mirror image_13 deleted '' 2026-02-22T22:05:15.971 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:15.971 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:15.971 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_13 2026-02-22T22:05:15.971 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=deleted 2026-02-22T22:05:15.971 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:05:15.971 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=deleted 2026-02-22T22:05:15.971 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_image_id 2026-02-22T22:05:15.971 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_13 2026-02-22T22:05:15.971 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:05:15.971 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:05:15.971 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_13 2026-02-22T22:05:15.971 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_13 2026-02-22T22:05:15.972 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:05:15.993 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_13: (2) No such file or directory 2026-02-22T22:05:15.995 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_image_id= 2026-02-22T22:05:15.995 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:05:15.996 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test deleted = deleted 2026-02-22T22:05:15.996 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:05:15.996 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:05:15.996 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_14 2026-02-22T22:05:15.996 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_present cluster1 mirror image_14 deleted 2026-02-22T22:05:15.996 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:15.996 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:15.996 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_14 2026-02-22T22:05:15.996 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=deleted 2026-02-22T22:05:15.996 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:05:15.996 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:05:15.996 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:05:15.996 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_14 2026-02-22T22:05:15.996 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:05:15.996 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:05:15.996 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_14 2026-02-22T22:05:15.996 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_14 2026-02-22T22:05:15.996 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:05:16.017 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_14: (2) No such file or directory 2026-02-22T22:05:16.018 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image_id= 2026-02-22T22:05:16.018 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:05:16.018 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.1 2026-02-22T22:05:16.120 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_present cluster1 mirror image_14 deleted '' 2026-02-22T22:05:16.120 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:16.120 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:16.120 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_14 2026-02-22T22:05:16.120 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=deleted 2026-02-22T22:05:16.120 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:05:16.120 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=deleted 2026-02-22T22:05:16.120 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_image_id 2026-02-22T22:05:16.120 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_14 2026-02-22T22:05:16.120 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:05:16.120 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:05:16.120 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_14 2026-02-22T22:05:16.121 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_14 2026-02-22T22:05:16.121 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:05:16.143 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_14: (2) No such file or directory 2026-02-22T22:05:16.145 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_image_id= 2026-02-22T22:05:16.145 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:05:16.145 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test deleted = deleted 2026-02-22T22:05:16.145 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:05:16.145 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:05:16.145 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_15 2026-02-22T22:05:16.145 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_present cluster1 mirror image_15 deleted 2026-02-22T22:05:16.145 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:16.145 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:16.145 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_15 2026-02-22T22:05:16.145 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=deleted 2026-02-22T22:05:16.145 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:05:16.145 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:05:16.145 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:05:16.145 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_15 2026-02-22T22:05:16.145 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:05:16.145 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:05:16.145 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_15 2026-02-22T22:05:16.145 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_15 2026-02-22T22:05:16.146 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:05:16.369 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_15: (2) No such file or directory 2026-02-22T22:05:16.372 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image_id= 2026-02-22T22:05:16.372 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:05:16.373 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.1 2026-02-22T22:05:16.474 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_present cluster1 mirror image_15 deleted '' 2026-02-22T22:05:16.474 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:16.474 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:16.474 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_15 2026-02-22T22:05:16.474 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=deleted 2026-02-22T22:05:16.474 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:05:16.474 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=deleted 2026-02-22T22:05:16.474 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_image_id 2026-02-22T22:05:16.474 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_15 2026-02-22T22:05:16.474 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:05:16.474 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:05:16.474 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_15 2026-02-22T22:05:16.474 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_15 2026-02-22T22:05:16.475 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:05:16.498 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_15: (2) No such file or directory 2026-02-22T22:05:16.500 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_image_id= 2026-02-22T22:05:16.500 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:05:16.500 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test deleted = deleted 2026-02-22T22:05:16.500 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:05:16.500 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:05:16.500 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_16 2026-02-22T22:05:16.500 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_present cluster1 mirror image_16 deleted 2026-02-22T22:05:16.500 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:16.500 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:16.500 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_16 2026-02-22T22:05:16.500 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=deleted 2026-02-22T22:05:16.500 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:05:16.500 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:05:16.500 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:05:16.500 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_16 2026-02-22T22:05:16.500 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:05:16.500 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:05:16.500 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_16 2026-02-22T22:05:16.500 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_16 2026-02-22T22:05:16.500 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:05:16.521 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_16: (2) No such file or directory 2026-02-22T22:05:16.523 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image_id= 2026-02-22T22:05:16.523 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:05:16.523 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.1 2026-02-22T22:05:16.624 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_present cluster1 mirror image_16 deleted '' 2026-02-22T22:05:16.624 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:16.624 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:16.624 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_16 2026-02-22T22:05:16.624 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=deleted 2026-02-22T22:05:16.624 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:05:16.624 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=deleted 2026-02-22T22:05:16.624 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_image_id 2026-02-22T22:05:16.624 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_16 2026-02-22T22:05:16.624 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:05:16.624 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:05:16.624 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_16 2026-02-22T22:05:16.625 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_16 2026-02-22T22:05:16.625 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:05:16.646 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_16: (2) No such file or directory 2026-02-22T22:05:16.648 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_image_id= 2026-02-22T22:05:16.648 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:05:16.648 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test deleted = deleted 2026-02-22T22:05:16.648 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:05:16.648 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:05:16.648 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_17 2026-02-22T22:05:16.648 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_present cluster1 mirror image_17 deleted 2026-02-22T22:05:16.648 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:16.648 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:16.648 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_17 2026-02-22T22:05:16.648 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=deleted 2026-02-22T22:05:16.648 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:05:16.648 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:05:16.648 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:05:16.648 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_17 2026-02-22T22:05:16.648 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:05:16.648 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:05:16.648 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_17 2026-02-22T22:05:16.649 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_17 2026-02-22T22:05:16.649 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:05:16.669 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_17: (2) No such file or directory 2026-02-22T22:05:16.670 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image_id= 2026-02-22T22:05:16.671 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:05:16.671 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.1 2026-02-22T22:05:16.771 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_present cluster1 mirror image_17 deleted '' 2026-02-22T22:05:16.771 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:16.771 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:16.771 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_17 2026-02-22T22:05:16.771 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=deleted 2026-02-22T22:05:16.771 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:05:16.771 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=deleted 2026-02-22T22:05:16.771 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_image_id 2026-02-22T22:05:16.772 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_17 2026-02-22T22:05:16.772 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:05:16.772 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:05:16.772 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_17 2026-02-22T22:05:16.772 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_17 2026-02-22T22:05:16.772 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:05:16.792 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_17: (2) No such file or directory 2026-02-22T22:05:16.794 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_image_id= 2026-02-22T22:05:16.794 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:05:16.794 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test deleted = deleted 2026-02-22T22:05:16.794 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:05:16.794 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:05:16.794 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_18 2026-02-22T22:05:16.794 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_present cluster1 mirror image_18 deleted 2026-02-22T22:05:16.794 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:16.794 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:16.794 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_18 2026-02-22T22:05:16.794 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=deleted 2026-02-22T22:05:16.794 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:05:16.794 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:05:16.795 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:05:16.795 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_18 2026-02-22T22:05:16.795 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:05:16.795 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:05:16.795 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_18 2026-02-22T22:05:16.795 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_18 2026-02-22T22:05:16.795 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:05:16.816 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_18: (2) No such file or directory 2026-02-22T22:05:16.817 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image_id= 2026-02-22T22:05:16.818 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:05:16.818 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.1 2026-02-22T22:05:16.919 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_present cluster1 mirror image_18 deleted '' 2026-02-22T22:05:16.919 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:16.919 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:16.919 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_18 2026-02-22T22:05:16.919 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=deleted 2026-02-22T22:05:16.919 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:05:16.919 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=deleted 2026-02-22T22:05:16.919 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_image_id 2026-02-22T22:05:16.919 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_18 2026-02-22T22:05:16.919 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:05:16.919 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:05:16.919 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_18 2026-02-22T22:05:16.919 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_18 2026-02-22T22:05:16.919 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:05:16.942 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_18: (2) No such file or directory 2026-02-22T22:05:16.946 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_image_id= 2026-02-22T22:05:16.946 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:05:16.946 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test deleted = deleted 2026-02-22T22:05:16.946 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:05:16.946 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:05:16.946 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_19 2026-02-22T22:05:16.946 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_present cluster1 mirror image_19 deleted 2026-02-22T22:05:16.946 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:16.947 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:16.947 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_19 2026-02-22T22:05:16.947 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=deleted 2026-02-22T22:05:16.947 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:05:16.947 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:05:16.947 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:05:16.947 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_19 2026-02-22T22:05:16.947 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:05:16.947 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:05:16.947 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_19 2026-02-22T22:05:16.947 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_19 2026-02-22T22:05:16.948 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:05:16.966 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_19: (2) No such file or directory 2026-02-22T22:05:16.968 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image_id= 2026-02-22T22:05:16.968 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:05:16.968 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.1 2026-02-22T22:05:17.069 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_present cluster1 mirror image_19 deleted '' 2026-02-22T22:05:17.070 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:17.070 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:17.070 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_19 2026-02-22T22:05:17.070 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=deleted 2026-02-22T22:05:17.070 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:05:17.070 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=deleted 2026-02-22T22:05:17.070 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_image_id 2026-02-22T22:05:17.070 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_19 2026-02-22T22:05:17.070 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:05:17.071 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:05:17.071 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_19 2026-02-22T22:05:17.071 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_19 2026-02-22T22:05:17.071 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:05:17.091 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_19: (2) No such file or directory 2026-02-22T22:05:17.093 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_image_id= 2026-02-22T22:05:17.094 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:05:17.094 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test deleted = deleted 2026-02-22T22:05:17.094 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:05:17.094 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:05:17.094 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_20 2026-02-22T22:05:17.094 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_present cluster1 mirror image_20 deleted 2026-02-22T22:05:17.094 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:17.094 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:17.094 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_20 2026-02-22T22:05:17.094 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=deleted 2026-02-22T22:05:17.094 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:05:17.094 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:05:17.094 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:05:17.094 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_20 2026-02-22T22:05:17.094 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:05:17.094 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:05:17.094 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_20 2026-02-22T22:05:17.094 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_20 2026-02-22T22:05:17.095 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:05:17.115 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_20: (2) No such file or directory 2026-02-22T22:05:17.117 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image_id= 2026-02-22T22:05:17.117 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:05:17.117 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.1 2026-02-22T22:05:17.219 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_present cluster1 mirror image_20 deleted '' 2026-02-22T22:05:17.219 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:17.219 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:17.219 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_20 2026-02-22T22:05:17.219 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=deleted 2026-02-22T22:05:17.219 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:05:17.220 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=deleted 2026-02-22T22:05:17.220 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_image_id 2026-02-22T22:05:17.220 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_20 2026-02-22T22:05:17.220 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:05:17.220 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:05:17.220 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_20 2026-02-22T22:05:17.220 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_20 2026-02-22T22:05:17.220 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:05:17.241 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_20: (2) No such file or directory 2026-02-22T22:05:17.244 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_image_id= 2026-02-22T22:05:17.244 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:05:17.245 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test deleted = deleted 2026-02-22T22:05:17.245 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:05:17.245 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:05:17.245 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_21 2026-02-22T22:05:17.245 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_present cluster1 mirror image_21 deleted 2026-02-22T22:05:17.245 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:17.245 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:17.245 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_21 2026-02-22T22:05:17.245 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=deleted 2026-02-22T22:05:17.245 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:05:17.245 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:05:17.245 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:05:17.246 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_21 2026-02-22T22:05:17.246 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:05:17.246 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:05:17.246 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_21 2026-02-22T22:05:17.246 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_21 2026-02-22T22:05:17.246 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:05:17.266 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_21: (2) No such file or directory 2026-02-22T22:05:17.268 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image_id= 2026-02-22T22:05:17.268 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:05:17.268 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.1 2026-02-22T22:05:17.369 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_present cluster1 mirror image_21 deleted '' 2026-02-22T22:05:17.370 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:17.370 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:17.370 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_21 2026-02-22T22:05:17.370 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=deleted 2026-02-22T22:05:17.370 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:05:17.370 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=deleted 2026-02-22T22:05:17.370 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_image_id 2026-02-22T22:05:17.370 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_21 2026-02-22T22:05:17.370 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:05:17.371 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:05:17.371 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_21 2026-02-22T22:05:17.371 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_21 2026-02-22T22:05:17.371 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:05:17.390 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_21: (2) No such file or directory 2026-02-22T22:05:17.393 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_image_id= 2026-02-22T22:05:17.393 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:05:17.393 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test deleted = deleted 2026-02-22T22:05:17.393 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:05:17.393 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:05:17.393 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_22 2026-02-22T22:05:17.393 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_present cluster1 mirror image_22 deleted 2026-02-22T22:05:17.394 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:17.394 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:17.394 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_22 2026-02-22T22:05:17.394 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=deleted 2026-02-22T22:05:17.394 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:05:17.394 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:05:17.394 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:05:17.394 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_22 2026-02-22T22:05:17.394 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:05:17.394 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:05:17.394 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_22 2026-02-22T22:05:17.394 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_22 2026-02-22T22:05:17.395 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:05:17.415 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_22: (2) No such file or directory 2026-02-22T22:05:17.417 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image_id= 2026-02-22T22:05:17.417 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:05:17.418 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.1 2026-02-22T22:05:17.518 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_present cluster1 mirror image_22 deleted '' 2026-02-22T22:05:17.519 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:17.519 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:17.519 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_22 2026-02-22T22:05:17.519 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=deleted 2026-02-22T22:05:17.519 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:05:17.519 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=deleted 2026-02-22T22:05:17.519 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_image_id 2026-02-22T22:05:17.519 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_22 2026-02-22T22:05:17.519 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:05:17.519 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:05:17.519 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_22 2026-02-22T22:05:17.520 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_22 2026-02-22T22:05:17.520 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:05:17.546 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_22: (2) No such file or directory 2026-02-22T22:05:17.549 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_image_id= 2026-02-22T22:05:17.549 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:05:17.550 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test deleted = deleted 2026-02-22T22:05:17.550 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:05:17.550 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:05:17.550 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_23 2026-02-22T22:05:17.550 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_present cluster1 mirror image_23 deleted 2026-02-22T22:05:17.550 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:17.550 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:17.550 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_23 2026-02-22T22:05:17.550 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=deleted 2026-02-22T22:05:17.550 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:05:17.550 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:05:17.551 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:05:17.551 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_23 2026-02-22T22:05:17.551 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:05:17.551 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:05:17.551 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_23 2026-02-22T22:05:17.551 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_23 2026-02-22T22:05:17.551 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:05:17.570 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_23: (2) No such file or directory 2026-02-22T22:05:17.572 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image_id= 2026-02-22T22:05:17.572 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:05:17.572 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.1 2026-02-22T22:05:17.673 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_present cluster1 mirror image_23 deleted '' 2026-02-22T22:05:17.673 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:17.673 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:17.673 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_23 2026-02-22T22:05:17.673 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=deleted 2026-02-22T22:05:17.673 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:05:17.673 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=deleted 2026-02-22T22:05:17.673 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_image_id 2026-02-22T22:05:17.673 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_23 2026-02-22T22:05:17.673 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:05:17.673 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:05:17.673 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_23 2026-02-22T22:05:17.674 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_23 2026-02-22T22:05:17.674 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:05:17.697 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_23: (2) No such file or directory 2026-02-22T22:05:17.700 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_image_id= 2026-02-22T22:05:17.701 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:05:17.701 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test deleted = deleted 2026-02-22T22:05:17.701 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:05:17.701 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:05:17.701 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_24 2026-02-22T22:05:17.701 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_present cluster1 mirror image_24 deleted 2026-02-22T22:05:17.701 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:17.701 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:17.702 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_24 2026-02-22T22:05:17.702 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=deleted 2026-02-22T22:05:17.702 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:05:17.702 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:05:17.702 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:05:17.702 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_24 2026-02-22T22:05:17.702 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:05:17.702 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:05:17.702 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_24 2026-02-22T22:05:17.702 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_24 2026-02-22T22:05:17.702 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:05:17.720 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_24: (2) No such file or directory 2026-02-22T22:05:17.722 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image_id= 2026-02-22T22:05:17.722 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:05:17.722 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.1 2026-02-22T22:05:17.824 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_present cluster1 mirror image_24 deleted '' 2026-02-22T22:05:17.824 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:17.824 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:17.824 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_24 2026-02-22T22:05:17.824 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=deleted 2026-02-22T22:05:17.825 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:05:17.825 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=deleted 2026-02-22T22:05:17.825 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_image_id 2026-02-22T22:05:17.825 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_24 2026-02-22T22:05:17.825 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:05:17.825 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:05:17.825 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_24 2026-02-22T22:05:17.825 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_24 2026-02-22T22:05:17.825 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:05:17.844 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_24: (2) No such file or directory 2026-02-22T22:05:17.846 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_image_id= 2026-02-22T22:05:17.846 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:05:17.846 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test deleted = deleted 2026-02-22T22:05:17.846 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:05:17.847 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:05:17.847 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_25 2026-02-22T22:05:17.847 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_present cluster1 mirror image_25 deleted 2026-02-22T22:05:17.847 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:17.847 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:17.847 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_25 2026-02-22T22:05:17.847 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=deleted 2026-02-22T22:05:17.847 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:05:17.847 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:05:17.847 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:05:17.848 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_25 2026-02-22T22:05:17.848 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:05:17.848 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:05:17.848 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_25 2026-02-22T22:05:17.848 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_25 2026-02-22T22:05:17.848 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:05:17.868 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_25: (2) No such file or directory 2026-02-22T22:05:17.869 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image_id= 2026-02-22T22:05:17.870 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:05:17.870 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.1 2026-02-22T22:05:17.971 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_present cluster1 mirror image_25 deleted '' 2026-02-22T22:05:17.971 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:17.971 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:17.972 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_25 2026-02-22T22:05:17.972 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=deleted 2026-02-22T22:05:17.972 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:05:17.972 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=deleted 2026-02-22T22:05:17.972 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_image_id 2026-02-22T22:05:17.972 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_25 2026-02-22T22:05:17.972 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:05:17.972 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:05:17.972 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_25 2026-02-22T22:05:17.973 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_25 2026-02-22T22:05:17.973 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:05:17.992 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_25: (2) No such file or directory 2026-02-22T22:05:17.994 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_image_id= 2026-02-22T22:05:17.994 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:05:17.994 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test deleted = deleted 2026-02-22T22:05:17.994 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:05:17.994 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:05:17.994 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_26 2026-02-22T22:05:17.994 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_present cluster1 mirror image_26 deleted 2026-02-22T22:05:17.994 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:17.994 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:17.994 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_26 2026-02-22T22:05:17.994 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=deleted 2026-02-22T22:05:17.994 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:05:17.994 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:05:17.994 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:05:17.995 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_26 2026-02-22T22:05:17.995 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:05:17.995 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:05:17.995 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_26 2026-02-22T22:05:17.995 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_26 2026-02-22T22:05:17.995 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:05:18.015 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_26: (2) No such file or directory 2026-02-22T22:05:18.017 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image_id= 2026-02-22T22:05:18.017 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:05:18.017 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.1 2026-02-22T22:05:18.118 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_present cluster1 mirror image_26 deleted '' 2026-02-22T22:05:18.118 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:18.119 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:18.119 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_26 2026-02-22T22:05:18.119 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=deleted 2026-02-22T22:05:18.119 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:05:18.119 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=deleted 2026-02-22T22:05:18.119 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_image_id 2026-02-22T22:05:18.119 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_26 2026-02-22T22:05:18.119 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:05:18.119 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:05:18.119 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_26 2026-02-22T22:05:18.120 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_26 2026-02-22T22:05:18.120 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:05:18.140 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_26: (2) No such file or directory 2026-02-22T22:05:18.143 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_image_id= 2026-02-22T22:05:18.143 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:05:18.143 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test deleted = deleted 2026-02-22T22:05:18.143 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:05:18.143 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:05:18.143 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_27 2026-02-22T22:05:18.143 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_present cluster1 mirror image_27 deleted 2026-02-22T22:05:18.143 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:18.143 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:18.143 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_27 2026-02-22T22:05:18.144 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=deleted 2026-02-22T22:05:18.144 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:05:18.144 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:05:18.144 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:05:18.144 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_27 2026-02-22T22:05:18.144 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:05:18.144 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:05:18.144 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_27 2026-02-22T22:05:18.144 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_27 2026-02-22T22:05:18.144 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:05:18.366 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_27: (2) No such file or directory 2026-02-22T22:05:18.369 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image_id= 2026-02-22T22:05:18.369 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:05:18.369 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.1 2026-02-22T22:05:18.471 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_present cluster1 mirror image_27 deleted '' 2026-02-22T22:05:18.471 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:18.471 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:18.471 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_27 2026-02-22T22:05:18.471 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=deleted 2026-02-22T22:05:18.471 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:05:18.471 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=deleted 2026-02-22T22:05:18.472 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_image_id 2026-02-22T22:05:18.472 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_27 2026-02-22T22:05:18.472 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:05:18.472 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:05:18.472 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_27 2026-02-22T22:05:18.472 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_27 2026-02-22T22:05:18.472 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:05:18.492 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_27: (2) No such file or directory 2026-02-22T22:05:18.495 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_image_id= 2026-02-22T22:05:18.496 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:05:18.496 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test deleted = deleted 2026-02-22T22:05:18.496 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:05:18.496 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:05:18.496 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_28 2026-02-22T22:05:18.496 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_present cluster1 mirror image_28 deleted 2026-02-22T22:05:18.496 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:18.496 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:18.496 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_28 2026-02-22T22:05:18.496 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=deleted 2026-02-22T22:05:18.497 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:05:18.497 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:05:18.497 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:05:18.497 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_28 2026-02-22T22:05:18.497 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:05:18.497 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:05:18.497 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_28 2026-02-22T22:05:18.497 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_28 2026-02-22T22:05:18.497 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:05:18.517 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_28: (2) No such file or directory 2026-02-22T22:05:18.519 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image_id= 2026-02-22T22:05:18.519 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:05:18.519 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.1 2026-02-22T22:05:18.620 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_present cluster1 mirror image_28 deleted '' 2026-02-22T22:05:18.620 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:18.620 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:18.620 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_28 2026-02-22T22:05:18.620 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=deleted 2026-02-22T22:05:18.620 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:05:18.620 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=deleted 2026-02-22T22:05:18.620 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_image_id 2026-02-22T22:05:18.620 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_28 2026-02-22T22:05:18.620 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:05:18.620 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:05:18.620 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_28 2026-02-22T22:05:18.620 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_28 2026-02-22T22:05:18.621 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:05:18.642 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_28: (2) No such file or directory 2026-02-22T22:05:18.644 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_image_id= 2026-02-22T22:05:18.644 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:05:18.645 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test deleted = deleted 2026-02-22T22:05:18.645 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:05:18.645 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:05:18.645 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_29 2026-02-22T22:05:18.645 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_present cluster1 mirror image_29 deleted 2026-02-22T22:05:18.645 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:18.645 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:18.645 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_29 2026-02-22T22:05:18.645 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=deleted 2026-02-22T22:05:18.645 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:05:18.645 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:05:18.645 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:05:18.645 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_29 2026-02-22T22:05:18.645 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:05:18.645 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:05:18.645 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_29 2026-02-22T22:05:18.645 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_29 2026-02-22T22:05:18.645 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:05:18.667 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_29: (2) No such file or directory 2026-02-22T22:05:18.669 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image_id= 2026-02-22T22:05:18.669 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:05:18.669 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.1 2026-02-22T22:05:18.771 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_present cluster1 mirror image_29 deleted '' 2026-02-22T22:05:18.771 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:18.771 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:18.771 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_29 2026-02-22T22:05:18.771 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=deleted 2026-02-22T22:05:18.771 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:05:18.772 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=deleted 2026-02-22T22:05:18.772 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_image_id 2026-02-22T22:05:18.772 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_29 2026-02-22T22:05:18.772 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:05:18.772 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:05:18.772 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_29 2026-02-22T22:05:18.772 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_29 2026-02-22T22:05:18.772 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:05:18.793 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_29: (2) No such file or directory 2026-02-22T22:05:18.796 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_image_id= 2026-02-22T22:05:18.796 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:05:18.797 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test deleted = deleted 2026-02-22T22:05:18.797 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:05:18.797 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:05:18.797 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_30 2026-02-22T22:05:18.797 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_present cluster1 mirror image_30 deleted 2026-02-22T22:05:18.797 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:18.797 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:18.797 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_30 2026-02-22T22:05:18.797 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=deleted 2026-02-22T22:05:18.797 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:05:18.798 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:05:18.798 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:05:18.798 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_30 2026-02-22T22:05:18.798 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:05:18.798 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:05:18.798 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_30 2026-02-22T22:05:18.798 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_30 2026-02-22T22:05:18.798 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:05:18.818 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_30: (2) No such file or directory 2026-02-22T22:05:18.820 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image_id= 2026-02-22T22:05:18.820 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:05:18.820 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.1 2026-02-22T22:05:18.922 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_present cluster1 mirror image_30 deleted '' 2026-02-22T22:05:18.922 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:18.922 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:18.922 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_30 2026-02-22T22:05:18.922 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=deleted 2026-02-22T22:05:18.922 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:05:18.922 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=deleted 2026-02-22T22:05:18.922 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_image_id 2026-02-22T22:05:18.923 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_30 2026-02-22T22:05:18.923 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:05:18.923 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:05:18.923 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_30 2026-02-22T22:05:18.923 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_30 2026-02-22T22:05:18.923 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:05:18.943 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_30: (2) No such file or directory 2026-02-22T22:05:18.947 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_image_id= 2026-02-22T22:05:18.947 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:05:18.947 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test deleted = deleted 2026-02-22T22:05:18.947 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:05:18.947 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:05:18.948 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_31 2026-02-22T22:05:18.948 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_present cluster1 mirror image_31 deleted 2026-02-22T22:05:18.948 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:18.948 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:18.948 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_31 2026-02-22T22:05:18.948 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=deleted 2026-02-22T22:05:18.948 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:05:18.948 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:05:18.949 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:05:18.949 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_31 2026-02-22T22:05:18.949 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:05:18.949 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:05:18.949 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_31 2026-02-22T22:05:18.949 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_31 2026-02-22T22:05:18.949 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:05:18.969 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_31: (2) No such file or directory 2026-02-22T22:05:18.971 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image_id= 2026-02-22T22:05:18.971 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:05:18.971 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.1 2026-02-22T22:05:19.073 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_present cluster1 mirror image_31 deleted '' 2026-02-22T22:05:19.073 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:19.073 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:19.073 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_31 2026-02-22T22:05:19.074 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=deleted 2026-02-22T22:05:19.074 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:05:19.074 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=deleted 2026-02-22T22:05:19.074 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_image_id 2026-02-22T22:05:19.074 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_31 2026-02-22T22:05:19.074 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:05:19.074 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:05:19.074 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_31 2026-02-22T22:05:19.074 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_31 2026-02-22T22:05:19.074 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:05:19.095 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_31: (2) No such file or directory 2026-02-22T22:05:19.097 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_image_id= 2026-02-22T22:05:19.098 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:05:19.098 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test deleted = deleted 2026-02-22T22:05:19.098 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:05:19.098 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:05:19.098 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_32 2026-02-22T22:05:19.098 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_present cluster1 mirror image_32 deleted 2026-02-22T22:05:19.098 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:19.098 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:19.098 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_32 2026-02-22T22:05:19.098 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=deleted 2026-02-22T22:05:19.099 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:05:19.099 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:05:19.099 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:05:19.099 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_32 2026-02-22T22:05:19.099 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:05:19.099 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:05:19.099 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_32 2026-02-22T22:05:19.099 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_32 2026-02-22T22:05:19.099 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:05:19.119 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_32: (2) No such file or directory 2026-02-22T22:05:19.121 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image_id= 2026-02-22T22:05:19.121 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:05:19.121 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.1 2026-02-22T22:05:19.223 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_present cluster1 mirror image_32 deleted '' 2026-02-22T22:05:19.223 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:19.223 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:19.223 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_32 2026-02-22T22:05:19.223 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=deleted 2026-02-22T22:05:19.223 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:05:19.223 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=deleted 2026-02-22T22:05:19.224 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_image_id 2026-02-22T22:05:19.224 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_32 2026-02-22T22:05:19.224 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:05:19.224 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:05:19.224 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_32 2026-02-22T22:05:19.224 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_32 2026-02-22T22:05:19.224 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:05:19.245 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_32: (2) No such file or directory 2026-02-22T22:05:19.248 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_image_id= 2026-02-22T22:05:19.248 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:05:19.248 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test deleted = deleted 2026-02-22T22:05:19.249 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:05:19.249 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:05:19.249 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_33 2026-02-22T22:05:19.249 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_present cluster1 mirror image_33 deleted 2026-02-22T22:05:19.249 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:19.249 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:19.249 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_33 2026-02-22T22:05:19.249 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=deleted 2026-02-22T22:05:19.249 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:05:19.249 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:05:19.249 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:05:19.250 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_33 2026-02-22T22:05:19.250 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:05:19.250 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:05:19.250 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_33 2026-02-22T22:05:19.250 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_33 2026-02-22T22:05:19.250 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:05:19.269 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_33: (2) No such file or directory 2026-02-22T22:05:19.270 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image_id= 2026-02-22T22:05:19.270 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:05:19.270 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.1 2026-02-22T22:05:19.371 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_present cluster1 mirror image_33 deleted '' 2026-02-22T22:05:19.371 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:19.371 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:19.371 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_33 2026-02-22T22:05:19.371 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=deleted 2026-02-22T22:05:19.371 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:05:19.371 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=deleted 2026-02-22T22:05:19.371 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_image_id 2026-02-22T22:05:19.371 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_33 2026-02-22T22:05:19.371 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:05:19.371 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:05:19.371 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_33 2026-02-22T22:05:19.371 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_33 2026-02-22T22:05:19.371 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:05:19.393 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_33: (2) No such file or directory 2026-02-22T22:05:19.394 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_image_id= 2026-02-22T22:05:19.394 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:05:19.394 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test deleted = deleted 2026-02-22T22:05:19.394 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:05:19.395 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:05:19.395 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_34 2026-02-22T22:05:19.395 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_present cluster1 mirror image_34 deleted 2026-02-22T22:05:19.395 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:19.395 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:19.395 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_34 2026-02-22T22:05:19.395 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=deleted 2026-02-22T22:05:19.395 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:05:19.395 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:05:19.395 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:05:19.395 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_34 2026-02-22T22:05:19.395 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:05:19.395 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:05:19.395 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_34 2026-02-22T22:05:19.395 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_34 2026-02-22T22:05:19.395 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:05:19.416 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_34: (2) No such file or directory 2026-02-22T22:05:19.418 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image_id= 2026-02-22T22:05:19.418 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:05:19.418 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.1 2026-02-22T22:05:19.520 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_present cluster1 mirror image_34 deleted '' 2026-02-22T22:05:19.520 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:19.520 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:19.520 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_34 2026-02-22T22:05:19.520 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=deleted 2026-02-22T22:05:19.520 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:05:19.520 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=deleted 2026-02-22T22:05:19.521 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_image_id 2026-02-22T22:05:19.521 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_34 2026-02-22T22:05:19.521 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:05:19.521 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:05:19.521 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_34 2026-02-22T22:05:19.521 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_34 2026-02-22T22:05:19.521 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:05:19.542 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_34: (2) No such file or directory 2026-02-22T22:05:19.544 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_image_id= 2026-02-22T22:05:19.545 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:05:19.545 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test deleted = deleted 2026-02-22T22:05:19.545 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:05:19.545 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:05:19.545 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_35 2026-02-22T22:05:19.545 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_present cluster1 mirror image_35 deleted 2026-02-22T22:05:19.545 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:19.545 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:19.545 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_35 2026-02-22T22:05:19.546 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=deleted 2026-02-22T22:05:19.546 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:05:19.546 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:05:19.546 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:05:19.546 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_35 2026-02-22T22:05:19.546 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:05:19.546 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:05:19.546 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_35 2026-02-22T22:05:19.546 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_35 2026-02-22T22:05:19.546 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:05:19.566 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_35: (2) No such file or directory 2026-02-22T22:05:19.568 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image_id= 2026-02-22T22:05:19.568 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:05:19.568 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.1 2026-02-22T22:05:19.670 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_present cluster1 mirror image_35 deleted '' 2026-02-22T22:05:19.671 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:19.671 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:19.671 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_35 2026-02-22T22:05:19.671 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=deleted 2026-02-22T22:05:19.671 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:05:19.671 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=deleted 2026-02-22T22:05:19.671 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_image_id 2026-02-22T22:05:19.671 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_35 2026-02-22T22:05:19.671 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:05:19.672 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:05:19.672 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_35 2026-02-22T22:05:19.672 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_35 2026-02-22T22:05:19.672 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:05:19.692 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_35: (2) No such file or directory 2026-02-22T22:05:19.696 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_image_id= 2026-02-22T22:05:19.697 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:05:19.697 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test deleted = deleted 2026-02-22T22:05:19.697 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:05:19.697 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:05:19.697 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_36 2026-02-22T22:05:19.698 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_present cluster1 mirror image_36 deleted 2026-02-22T22:05:19.698 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:19.698 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:19.698 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_36 2026-02-22T22:05:19.698 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=deleted 2026-02-22T22:05:19.698 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:05:19.699 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:05:19.699 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:05:19.699 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_36 2026-02-22T22:05:19.699 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:05:19.699 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:05:19.699 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_36 2026-02-22T22:05:19.700 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_36 2026-02-22T22:05:19.700 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:05:19.716 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_36: (2) No such file or directory 2026-02-22T22:05:19.719 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image_id= 2026-02-22T22:05:19.719 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:05:19.719 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.1 2026-02-22T22:05:19.821 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_present cluster1 mirror image_36 deleted '' 2026-02-22T22:05:19.821 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:19.821 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:19.821 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_36 2026-02-22T22:05:19.821 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=deleted 2026-02-22T22:05:19.821 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:05:19.821 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=deleted 2026-02-22T22:05:19.821 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_image_id 2026-02-22T22:05:19.822 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_36 2026-02-22T22:05:19.822 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:05:19.822 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:05:19.822 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_36 2026-02-22T22:05:19.822 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_36 2026-02-22T22:05:19.822 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:05:19.843 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_36: (2) No such file or directory 2026-02-22T22:05:19.845 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_image_id= 2026-02-22T22:05:19.846 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:05:19.846 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test deleted = deleted 2026-02-22T22:05:19.846 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:05:19.846 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:05:19.846 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_37 2026-02-22T22:05:19.846 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_present cluster1 mirror image_37 deleted 2026-02-22T22:05:19.846 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:19.846 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:19.846 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_37 2026-02-22T22:05:19.846 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=deleted 2026-02-22T22:05:19.846 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:05:19.846 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:05:19.846 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:05:19.846 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_37 2026-02-22T22:05:19.846 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:05:19.846 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:05:19.846 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_37 2026-02-22T22:05:19.846 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_37 2026-02-22T22:05:19.846 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:05:19.869 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_37: (2) No such file or directory 2026-02-22T22:05:19.871 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image_id= 2026-02-22T22:05:19.871 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:05:19.871 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.1 2026-02-22T22:05:19.972 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_present cluster1 mirror image_37 deleted '' 2026-02-22T22:05:19.973 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:19.973 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:19.973 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_37 2026-02-22T22:05:19.973 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=deleted 2026-02-22T22:05:19.973 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:05:19.973 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=deleted 2026-02-22T22:05:19.973 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_image_id 2026-02-22T22:05:19.973 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_37 2026-02-22T22:05:19.973 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:05:19.973 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:05:19.974 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_37 2026-02-22T22:05:19.974 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_37 2026-02-22T22:05:19.974 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:05:19.994 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_37: (2) No such file or directory 2026-02-22T22:05:19.997 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_image_id= 2026-02-22T22:05:19.997 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:05:19.997 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test deleted = deleted 2026-02-22T22:05:19.997 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:05:19.997 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:05:19.997 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_38 2026-02-22T22:05:19.997 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_present cluster1 mirror image_38 deleted 2026-02-22T22:05:19.997 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:19.997 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:19.997 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_38 2026-02-22T22:05:19.998 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=deleted 2026-02-22T22:05:19.998 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:05:19.998 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:05:19.998 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:05:19.998 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_38 2026-02-22T22:05:19.998 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:05:19.998 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:05:19.998 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_38 2026-02-22T22:05:19.998 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_38 2026-02-22T22:05:19.998 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:05:20.018 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_38: (2) No such file or directory 2026-02-22T22:05:20.019 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image_id= 2026-02-22T22:05:20.020 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:05:20.020 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.1 2026-02-22T22:05:20.120 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_present cluster1 mirror image_38 deleted '' 2026-02-22T22:05:20.120 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:20.120 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:20.120 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_38 2026-02-22T22:05:20.120 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=deleted 2026-02-22T22:05:20.120 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:05:20.120 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=deleted 2026-02-22T22:05:20.120 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_image_id 2026-02-22T22:05:20.120 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_38 2026-02-22T22:05:20.120 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:05:20.120 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:05:20.120 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_38 2026-02-22T22:05:20.120 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_38 2026-02-22T22:05:20.121 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:05:20.143 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_38: (2) No such file or directory 2026-02-22T22:05:20.145 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_image_id= 2026-02-22T22:05:20.145 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:05:20.145 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test deleted = deleted 2026-02-22T22:05:20.145 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:05:20.145 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:05:20.145 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_39 2026-02-22T22:05:20.145 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_present cluster1 mirror image_39 deleted 2026-02-22T22:05:20.145 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:20.145 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:20.145 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_39 2026-02-22T22:05:20.145 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=deleted 2026-02-22T22:05:20.145 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:05:20.145 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:05:20.145 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:05:20.145 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_39 2026-02-22T22:05:20.145 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:05:20.146 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:05:20.146 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_39 2026-02-22T22:05:20.146 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_39 2026-02-22T22:05:20.146 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:05:20.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_39: (2) No such file or directory 2026-02-22T22:05:20.170 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image_id= 2026-02-22T22:05:20.170 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:05:20.171 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.1 2026-02-22T22:05:20.270 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_present cluster1 mirror image_39 deleted '' 2026-02-22T22:05:20.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:20.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:20.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_39 2026-02-22T22:05:20.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=deleted 2026-02-22T22:05:20.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:05:20.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=deleted 2026-02-22T22:05:20.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_image_id 2026-02-22T22:05:20.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_39 2026-02-22T22:05:20.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:05:20.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:05:20.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_39 2026-02-22T22:05:20.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_39 2026-02-22T22:05:20.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:05:20.293 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_39: (2) No such file or directory 2026-02-22T22:05:20.296 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_image_id= 2026-02-22T22:05:20.296 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:05:20.296 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test deleted = deleted 2026-02-22T22:05:20.296 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:05:20.297 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:05:20.297 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_40 2026-02-22T22:05:20.297 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_present cluster1 mirror image_40 deleted 2026-02-22T22:05:20.297 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:20.297 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:20.297 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_40 2026-02-22T22:05:20.297 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=deleted 2026-02-22T22:05:20.297 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:05:20.297 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:05:20.297 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:05:20.297 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_40 2026-02-22T22:05:20.298 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:05:20.298 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:05:20.298 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_40 2026-02-22T22:05:20.298 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_40 2026-02-22T22:05:20.298 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:05:20.319 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_40: (2) No such file or directory 2026-02-22T22:05:20.321 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image_id= 2026-02-22T22:05:20.321 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:05:20.321 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.1 2026-02-22T22:05:20.422 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_present cluster1 mirror image_40 deleted '' 2026-02-22T22:05:20.422 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:20.422 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:20.422 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_40 2026-02-22T22:05:20.422 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=deleted 2026-02-22T22:05:20.422 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:05:20.422 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=deleted 2026-02-22T22:05:20.422 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_image_id 2026-02-22T22:05:20.423 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_40 2026-02-22T22:05:20.423 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:05:20.423 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:05:20.423 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_40 2026-02-22T22:05:20.423 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_40 2026-02-22T22:05:20.423 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:05:20.444 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_40: (2) No such file or directory 2026-02-22T22:05:20.445 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_image_id= 2026-02-22T22:05:20.446 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:05:20.446 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test deleted = deleted 2026-02-22T22:05:20.446 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:05:20.446 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:05:20.446 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_41 2026-02-22T22:05:20.446 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_present cluster1 mirror image_41 deleted 2026-02-22T22:05:20.446 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:20.446 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:20.446 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_41 2026-02-22T22:05:20.446 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=deleted 2026-02-22T22:05:20.446 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:05:20.446 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:05:20.446 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:05:20.446 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_41 2026-02-22T22:05:20.446 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:05:20.446 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:05:20.446 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_41 2026-02-22T22:05:20.446 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_41 2026-02-22T22:05:20.446 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:05:20.468 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_41: (2) No such file or directory 2026-02-22T22:05:20.470 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image_id= 2026-02-22T22:05:20.470 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:05:20.470 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.1 2026-02-22T22:05:20.572 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_present cluster1 mirror image_41 deleted '' 2026-02-22T22:05:20.573 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:20.573 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:20.573 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_41 2026-02-22T22:05:20.573 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=deleted 2026-02-22T22:05:20.573 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:05:20.573 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=deleted 2026-02-22T22:05:20.573 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_image_id 2026-02-22T22:05:20.573 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_41 2026-02-22T22:05:20.574 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:05:20.574 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:05:20.574 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_41 2026-02-22T22:05:20.574 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_41 2026-02-22T22:05:20.574 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:05:20.593 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_41: (2) No such file or directory 2026-02-22T22:05:20.595 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_image_id= 2026-02-22T22:05:20.595 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:05:20.595 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test deleted = deleted 2026-02-22T22:05:20.595 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:05:20.595 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:05:20.595 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_42 2026-02-22T22:05:20.595 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_present cluster1 mirror image_42 deleted 2026-02-22T22:05:20.595 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:20.595 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:20.595 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_42 2026-02-22T22:05:20.595 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=deleted 2026-02-22T22:05:20.595 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:05:20.595 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:05:20.595 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:05:20.595 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_42 2026-02-22T22:05:20.595 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:05:20.596 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:05:20.596 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_42 2026-02-22T22:05:20.596 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_42 2026-02-22T22:05:20.596 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:05:20.616 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_42: (2) No such file or directory 2026-02-22T22:05:20.618 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image_id= 2026-02-22T22:05:20.618 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:05:20.618 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.1 2026-02-22T22:05:20.720 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_present cluster1 mirror image_42 deleted '' 2026-02-22T22:05:20.720 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:20.720 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:20.720 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_42 2026-02-22T22:05:20.720 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=deleted 2026-02-22T22:05:20.721 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:05:20.721 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=deleted 2026-02-22T22:05:20.721 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_image_id 2026-02-22T22:05:20.721 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_42 2026-02-22T22:05:20.721 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:05:20.721 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:05:20.721 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_42 2026-02-22T22:05:20.721 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_42 2026-02-22T22:05:20.721 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:05:20.741 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_42: (2) No such file or directory 2026-02-22T22:05:20.742 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_image_id= 2026-02-22T22:05:20.743 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:05:20.743 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test deleted = deleted 2026-02-22T22:05:20.743 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:05:20.743 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:05:20.743 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_43 2026-02-22T22:05:20.743 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_present cluster1 mirror image_43 deleted 2026-02-22T22:05:20.743 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:20.743 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:20.743 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_43 2026-02-22T22:05:20.743 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=deleted 2026-02-22T22:05:20.743 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:05:20.743 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:05:20.743 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:05:20.743 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_43 2026-02-22T22:05:20.743 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:05:20.743 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:05:20.743 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_43 2026-02-22T22:05:20.743 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_43 2026-02-22T22:05:20.743 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:05:20.766 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_43: (2) No such file or directory 2026-02-22T22:05:20.767 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image_id= 2026-02-22T22:05:20.767 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:05:20.768 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.1 2026-02-22T22:05:20.868 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_present cluster1 mirror image_43 deleted '' 2026-02-22T22:05:20.868 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:20.868 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:20.868 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_43 2026-02-22T22:05:20.868 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=deleted 2026-02-22T22:05:20.868 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:05:20.868 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=deleted 2026-02-22T22:05:20.868 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_image_id 2026-02-22T22:05:20.868 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_43 2026-02-22T22:05:20.869 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:05:20.869 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:05:20.869 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_43 2026-02-22T22:05:20.869 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_43 2026-02-22T22:05:20.869 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:05:20.890 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_43: (2) No such file or directory 2026-02-22T22:05:20.892 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_image_id= 2026-02-22T22:05:20.893 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:05:20.893 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test deleted = deleted 2026-02-22T22:05:20.893 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:05:20.893 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:05:20.893 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_44 2026-02-22T22:05:20.893 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_present cluster1 mirror image_44 deleted 2026-02-22T22:05:20.893 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:20.893 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:20.893 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_44 2026-02-22T22:05:20.893 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=deleted 2026-02-22T22:05:20.893 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:05:20.893 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:05:20.893 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:05:20.893 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_44 2026-02-22T22:05:20.893 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:05:20.893 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:05:20.893 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_44 2026-02-22T22:05:20.893 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_44 2026-02-22T22:05:20.893 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:05:20.914 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_44: (2) No such file or directory 2026-02-22T22:05:20.916 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image_id= 2026-02-22T22:05:20.916 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:05:20.916 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.1 2026-02-22T22:05:21.018 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_present cluster1 mirror image_44 deleted '' 2026-02-22T22:05:21.018 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:21.018 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:21.018 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_44 2026-02-22T22:05:21.018 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=deleted 2026-02-22T22:05:21.018 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:05:21.018 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=deleted 2026-02-22T22:05:21.018 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_image_id 2026-02-22T22:05:21.018 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_44 2026-02-22T22:05:21.019 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:05:21.019 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:05:21.019 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_44 2026-02-22T22:05:21.019 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_44 2026-02-22T22:05:21.019 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:05:21.039 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_44: (2) No such file or directory 2026-02-22T22:05:21.042 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_image_id= 2026-02-22T22:05:21.042 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:05:21.042 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test deleted = deleted 2026-02-22T22:05:21.042 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:05:21.042 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:05:21.042 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_45 2026-02-22T22:05:21.042 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_present cluster1 mirror image_45 deleted 2026-02-22T22:05:21.042 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:21.042 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:21.042 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_45 2026-02-22T22:05:21.043 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=deleted 2026-02-22T22:05:21.043 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:05:21.043 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:05:21.043 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:05:21.043 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_45 2026-02-22T22:05:21.043 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:05:21.043 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:05:21.043 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_45 2026-02-22T22:05:21.043 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_45 2026-02-22T22:05:21.043 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:05:21.062 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_45: (2) No such file or directory 2026-02-22T22:05:21.064 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image_id= 2026-02-22T22:05:21.064 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:05:21.064 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.1 2026-02-22T22:05:21.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_present cluster1 mirror image_45 deleted '' 2026-02-22T22:05:21.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:21.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:21.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_45 2026-02-22T22:05:21.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=deleted 2026-02-22T22:05:21.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:05:21.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=deleted 2026-02-22T22:05:21.169 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_image_id 2026-02-22T22:05:21.169 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_45 2026-02-22T22:05:21.169 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:05:21.169 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:05:21.169 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_45 2026-02-22T22:05:21.169 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_45 2026-02-22T22:05:21.169 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:05:21.188 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_45: (2) No such file or directory 2026-02-22T22:05:21.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_image_id= 2026-02-22T22:05:21.191 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:05:21.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test deleted = deleted 2026-02-22T22:05:21.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:05:21.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:05:21.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_46 2026-02-22T22:05:21.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_present cluster1 mirror image_46 deleted 2026-02-22T22:05:21.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:21.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:21.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_46 2026-02-22T22:05:21.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=deleted 2026-02-22T22:05:21.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:05:21.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:05:21.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:05:21.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_46 2026-02-22T22:05:21.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:05:21.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:05:21.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_46 2026-02-22T22:05:21.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_46 2026-02-22T22:05:21.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:05:21.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_46: (2) No such file or directory 2026-02-22T22:05:21.213 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image_id= 2026-02-22T22:05:21.213 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:05:21.213 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.1 2026-02-22T22:05:21.314 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_present cluster1 mirror image_46 deleted '' 2026-02-22T22:05:21.314 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:21.314 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:21.314 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_46 2026-02-22T22:05:21.314 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=deleted 2026-02-22T22:05:21.314 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:05:21.314 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=deleted 2026-02-22T22:05:21.314 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_image_id 2026-02-22T22:05:21.314 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_46 2026-02-22T22:05:21.314 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:05:21.314 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:05:21.314 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_46 2026-02-22T22:05:21.314 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_46 2026-02-22T22:05:21.314 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:05:21.337 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_46: (2) No such file or directory 2026-02-22T22:05:21.339 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_image_id= 2026-02-22T22:05:21.339 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:05:21.339 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test deleted = deleted 2026-02-22T22:05:21.339 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:05:21.339 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:05:21.339 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_47 2026-02-22T22:05:21.339 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_present cluster1 mirror image_47 deleted 2026-02-22T22:05:21.339 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:21.339 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:21.339 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_47 2026-02-22T22:05:21.339 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=deleted 2026-02-22T22:05:21.339 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:05:21.339 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:05:21.339 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:05:21.340 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_47 2026-02-22T22:05:21.340 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:05:21.340 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:05:21.340 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_47 2026-02-22T22:05:21.340 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_47 2026-02-22T22:05:21.340 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:05:21.362 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_47: (2) No such file or directory 2026-02-22T22:05:21.363 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image_id= 2026-02-22T22:05:21.364 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:05:21.364 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.1 2026-02-22T22:05:21.466 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_present cluster1 mirror image_47 deleted '' 2026-02-22T22:05:21.466 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:21.466 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:21.466 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_47 2026-02-22T22:05:21.466 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=deleted 2026-02-22T22:05:21.466 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:05:21.466 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=deleted 2026-02-22T22:05:21.466 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_image_id 2026-02-22T22:05:21.467 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_47 2026-02-22T22:05:21.467 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:05:21.467 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:05:21.467 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_47 2026-02-22T22:05:21.467 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_47 2026-02-22T22:05:21.467 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:05:21.488 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_47: (2) No such file or directory 2026-02-22T22:05:21.491 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_image_id= 2026-02-22T22:05:21.492 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:05:21.492 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test deleted = deleted 2026-02-22T22:05:21.492 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:05:21.492 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:05:21.492 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_48 2026-02-22T22:05:21.492 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_present cluster1 mirror image_48 deleted 2026-02-22T22:05:21.492 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:21.492 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:21.492 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_48 2026-02-22T22:05:21.492 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=deleted 2026-02-22T22:05:21.492 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:05:21.492 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:05:21.493 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:05:21.493 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_48 2026-02-22T22:05:21.493 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:05:21.493 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:05:21.493 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_48 2026-02-22T22:05:21.493 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_48 2026-02-22T22:05:21.493 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:05:21.513 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_48: (2) No such file or directory 2026-02-22T22:05:21.515 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image_id= 2026-02-22T22:05:21.515 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:05:21.515 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.1 2026-02-22T22:05:21.617 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_present cluster1 mirror image_48 deleted '' 2026-02-22T22:05:21.617 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:21.617 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:21.617 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_48 2026-02-22T22:05:21.617 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=deleted 2026-02-22T22:05:21.618 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:05:21.618 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=deleted 2026-02-22T22:05:21.618 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_image_id 2026-02-22T22:05:21.618 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_48 2026-02-22T22:05:21.618 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:05:21.618 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:05:21.618 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_48 2026-02-22T22:05:21.618 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_48 2026-02-22T22:05:21.618 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:05:21.638 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_48: (2) No such file or directory 2026-02-22T22:05:21.641 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_image_id= 2026-02-22T22:05:21.641 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:05:21.642 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test deleted = deleted 2026-02-22T22:05:21.642 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:05:21.642 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:05:21.642 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_49 2026-02-22T22:05:21.642 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_present cluster1 mirror image_49 deleted 2026-02-22T22:05:21.642 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:21.642 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:21.642 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_49 2026-02-22T22:05:21.642 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=deleted 2026-02-22T22:05:21.642 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:05:21.642 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:05:21.642 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:05:21.643 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_49 2026-02-22T22:05:21.643 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:05:21.643 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:05:21.643 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_49 2026-02-22T22:05:21.643 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_49 2026-02-22T22:05:21.643 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:05:21.662 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_49: (2) No such file or directory 2026-02-22T22:05:21.663 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image_id= 2026-02-22T22:05:21.663 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:05:21.664 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.1 2026-02-22T22:05:21.766 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_present cluster1 mirror image_49 deleted '' 2026-02-22T22:05:21.766 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:21.766 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:21.766 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_49 2026-02-22T22:05:21.766 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=deleted 2026-02-22T22:05:21.767 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:05:21.767 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=deleted 2026-02-22T22:05:21.767 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_image_id 2026-02-22T22:05:21.767 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_49 2026-02-22T22:05:21.767 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:05:21.767 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:05:21.767 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_49 2026-02-22T22:05:21.767 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_49 2026-02-22T22:05:21.767 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:05:21.786 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_49: (2) No such file or directory 2026-02-22T22:05:21.789 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_image_id= 2026-02-22T22:05:21.789 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:05:21.789 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test deleted = deleted 2026-02-22T22:05:21.789 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:05:21.790 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-22T22:05:21.790 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_50 2026-02-22T22:05:21.790 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_present cluster1 mirror image_50 deleted 2026-02-22T22:05:21.790 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:21.790 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:21.790 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_50 2026-02-22T22:05:21.790 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=deleted 2026-02-22T22:05:21.790 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:05:21.790 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:05:21.790 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:05:21.790 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_50 2026-02-22T22:05:21.790 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:05:21.791 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:05:21.791 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_50 2026-02-22T22:05:21.791 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_50 2026-02-22T22:05:21.791 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:05:21.812 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_50: (2) No such file or directory 2026-02-22T22:05:21.814 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image_id= 2026-02-22T22:05:21.814 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:05:21.814 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.1 2026-02-22T22:05:21.915 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_present cluster1 mirror image_50 deleted '' 2026-02-22T22:05:21.915 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:21.915 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:21.915 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_50 2026-02-22T22:05:21.915 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=deleted 2026-02-22T22:05:21.915 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:05:21.915 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=deleted 2026-02-22T22:05:21.915 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_image_id 2026-02-22T22:05:21.915 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_50 2026-02-22T22:05:21.916 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:05:21.916 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:05:21.916 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_50 2026-02-22T22:05:21.916 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_50 2026-02-22T22:05:21.916 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:05:21.938 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_50: (2) No such file or directory 2026-02-22T22:05:21.940 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_image_id= 2026-02-22T22:05:21.940 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:05:21.940 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test deleted = deleted 2026-02-22T22:05:21.940 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:05:21.940 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ testlog 'TEST: delete images during bootstrap' 2026-02-22T22:05:21.941 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ tee -a /tmp/tmp.j44LZPowal/rbd-mirror.test.log 2026-02-22T22:05:21.941 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ date '+%F %T' 2026-02-22T22:05:21.941 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ echo 2026-02-22 22:05:21 TEST: delete images during bootstrap 2026-02-22T22:05:21.942 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:2026-02-22 22:05:21 TEST: delete images during bootstrap 2026-02-22T22:05:21.942 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set_pool_mirror_mode cluster1 mirror image 2026-02-22T22:05:21.942 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:21.942 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:21.942 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=image 2026-02-22T22:05:21.942 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster1 mirror pool enable mirror image 2026-02-22T22:05:21.962 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd: mirroring is already configured for image mode 2026-02-22T22:05:21.964 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set_pool_mirror_mode cluster2 mirror image 2026-02-22T22:05:21.964 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:05:21.964 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:21.964 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=image 2026-02-22T22:05:21.964 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 mirror pool enable mirror image 2026-02-22T22:05:21.985 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd: mirroring is already configured for image mode 2026-02-22T22:05:21.988 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ start_mirror cluster1 2026-02-22T22:05:21.988 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:21.989 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local instance 2026-02-22T22:05:21.989 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set_cluster_instance cluster1 cluster instance 2026-02-22T22:05:21.989 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local val=cluster1 2026-02-22T22:05:21.989 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster_var_name=cluster 2026-02-22T22:05:21.989 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local instance_var_name=instance 2026-02-22T22:05:21.989 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ cluster=cluster1 2026-02-22T22:05:21.989 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ instance=cluster1 2026-02-22T22:05:21.989 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' cluster1 = cluster1 ']' 2026-02-22T22:05:21.989 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ instance=0 2026-02-22T22:05:21.989 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ eval cluster=cluster1 2026-02-22T22:05:21.989 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ cluster=cluster1 2026-02-22T22:05:21.989 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ eval instance=0 2026-02-22T22:05:21.990 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ instance=0 2026-02-22T22:05:21.990 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n 1 2026-02-22T22:05:21.990 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 2026-02-22T22:05:21.990 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=test 2026-02-22T22:05:21.990 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ seq 1 10 2026-02-22T22:05:21.990 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 10` 2026-02-22T22:05:21.990 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_1 2026-02-22T22:05:21.990 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image cluster2 mirror image_1 512M 2026-02-22T22:05:21.990 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:05:21.990 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T22:05:21.990 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:21.990 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T22:05:21.990 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_1 2026-02-22T22:05:21.991 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T22:05:21.991 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local size=128 2026-02-22T22:05:21.991 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n 512M ']' 2026-02-22T22:05:21.991 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ size=512M 2026-02-22T22:05:21.991 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T22:05:21.991 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering,exclusive-lock mirror/image_1 2026-02-22T22:05:22.018 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ enable_mirror cluster2 mirror image_1 2026-02-22T22:05:22.019 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:05:22.019 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:22.019 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_1 2026-02-22T22:05:22.019 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T22:05:22.019 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_1 snapshot 2026-02-22T22:05:22.223 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Mirroring enabled 2026-02-22T22:05:22.230 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 info mirror/image_1 2026-02-22T22:05:22.254 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_1': 2026-02-22T22:05:22.254 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 512 MiB in 128 objects 2026-02-22T22:05:22.254 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T22:05:22.254 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 1 2026-02-22T22:05:22.254 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 18c76ed4fcd4 2026-02-22T22:05:22.254 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.18c76ed4fcd4 2026-02-22T22:05:22.254 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T22:05:22.254 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock 2026-02-22T22:05:22.254 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T22:05:22.254 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T22:05:22.254 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 22:05:22 2026 2026-02-22T22:05:22.254 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 22:05:22 2026 2026-02-22T22:05:22.254 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 22:05:22 2026 2026-02-22T22:05:22.254 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T22:05:22.254 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T22:05:22.254 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: 97b35fb1-1ed0-4b52-beb9-234ed156846b 2026-02-22T22:05:22.254 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: true 2026-02-22T22:05:22.255 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ stress_write_image cluster2 mirror image_1 2026-02-22T22:05:22.255 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:05:22.255 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:22.255 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_1 2026-02-22T22:05:22.256 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-22T22:05:22.257 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local duration=7 2026-02-22T22:05:22.257 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set +e 2026-02-22T22:05:22.257 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ timeout 7s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_1 --debug-rbd=20 --debug-journaler=20 2026-02-22T22:05:22.355 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:100 IOs, 84290560 bytes 2026-02-22T22:05:22.417 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:200 IOs, 160616448 bytes 2026-02-22T22:05:22.458 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:300 IOs, 194449408 bytes 2026-02-22T22:05:22.494 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:400 IOs, 211085312 bytes 2026-02-22T22:05:22.539 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:500 IOs, 256025600 bytes 2026-02-22T22:05:22.591 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:600 IOs, 310642688 bytes 2026-02-22T22:05:22.644 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:700 IOs, 384732160 bytes 2026-02-22T22:05:22.689 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:800 IOs, 425011200 bytes 2026-02-22T22:05:22.733 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:900 IOs, 466102272 bytes 2026-02-22T22:05:22.774 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1000 IOs, 487489536 bytes 2026-02-22T22:05:22.827 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1100 IOs, 549358592 bytes 2026-02-22T22:05:22.874 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1200 IOs, 597102592 bytes 2026-02-22T22:05:22.929 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1300 IOs, 673936384 bytes 2026-02-22T22:05:22.990 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1400 IOs, 746086400 bytes 2026-02-22T22:05:23.032 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1500 IOs, 777118720 bytes 2026-02-22T22:05:23.065 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1600 IOs, 793568256 bytes 2026-02-22T22:05:23.105 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1700 IOs, 813389824 bytes 2026-02-22T22:05:23.153 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1800 IOs, 889404416 bytes 2026-02-22T22:05:23.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1900 IOs, 937522176 bytes 2026-02-22T22:05:23.266 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2000 IOs, 1001359360 bytes 2026-02-22T22:05:23.323 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2100 IOs, 1071802368 bytes 2026-02-22T22:05:23.364 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2200 IOs, 1096119296 bytes 2026-02-22T22:05:23.408 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2300 IOs, 1136600064 bytes 2026-02-22T22:05:23.461 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2400 IOs, 1194281984 bytes 2026-02-22T22:05:23.500 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2500 IOs, 1235623936 bytes 2026-02-22T22:05:23.553 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2600 IOs, 1287939072 bytes 2026-02-22T22:05:23.607 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2700 IOs, 1339911168 bytes 2026-02-22T22:05:23.659 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2800 IOs, 1395639296 bytes 2026-02-22T22:05:23.717 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2900 IOs, 1466004480 bytes 2026-02-22T22:05:23.776 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3000 IOs, 1543376896 bytes 2026-02-22T22:05:23.834 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3100 IOs, 1612148736 bytes 2026-02-22T22:05:23.871 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3200 IOs, 1629341696 bytes 2026-02-22T22:05:23.920 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3300 IOs, 1676607488 bytes 2026-02-22T22:05:23.968 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3400 IOs, 1723650048 bytes 2026-02-22T22:05:24.004 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3500 IOs, 1738942464 bytes 2026-02-22T22:05:24.036 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3600 IOs, 1739567104 bytes 2026-02-22T22:05:24.076 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3700 IOs, 1762038784 bytes 2026-02-22T22:05:24.122 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3800 IOs, 1791822848 bytes 2026-02-22T22:05:24.154 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3900 IOs, 1793700864 bytes 2026-02-22T22:05:24.211 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4000 IOs, 1851674624 bytes 2026-02-22T22:05:24.261 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4100 IOs, 1897169920 bytes 2026-02-22T22:05:24.296 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4200 IOs, 1903759360 bytes 2026-02-22T22:05:24.337 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4300 IOs, 1933109248 bytes 2026-02-22T22:05:24.377 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4400 IOs, 1952889856 bytes 2026-02-22T22:05:24.422 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4500 IOs, 1987504128 bytes 2026-02-22T22:05:24.479 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4600 IOs, 2040039424 bytes 2026-02-22T22:05:24.537 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4700 IOs, 2098536448 bytes 2026-02-22T22:05:24.569 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4800 IOs, 2110088192 bytes 2026-02-22T22:05:24.614 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4900 IOs, 2155209728 bytes 2026-02-22T22:05:24.671 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5000 IOs, 2222510080 bytes 2026-02-22T22:05:24.706 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5100 IOs, 2234078208 bytes 2026-02-22T22:05:24.760 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5200 IOs, 2282976256 bytes 2026-02-22T22:05:24.828 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5300 IOs, 2370735104 bytes 2026-02-22T22:05:24.866 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5400 IOs, 2392564736 bytes 2026-02-22T22:05:24.920 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5500 IOs, 2447203328 bytes 2026-02-22T22:05:24.951 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5600 IOs, 2447813632 bytes 2026-02-22T22:05:24.996 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5700 IOs, 2490326016 bytes 2026-02-22T22:05:25.036 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5800 IOs, 2513087488 bytes 2026-02-22T22:05:25.073 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5900 IOs, 2527514624 bytes 2026-02-22T22:05:25.112 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6000 IOs, 2540073984 bytes 2026-02-22T22:05:25.148 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6100 IOs, 2557414400 bytes 2026-02-22T22:05:25.180 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6200 IOs, 2558012416 bytes 2026-02-22T22:05:25.247 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6300 IOs, 2646875136 bytes 2026-02-22T22:05:25.309 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6400 IOs, 2731287552 bytes 2026-02-22T22:05:25.321 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.5544608441027031) > weight (0.25) 2026-02-22T22:05:25.321 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.1 2026-02-22T22:05:25.321 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Sent signal 15 2026-02-22T22:05:25.321 INFO:tasks.rbd_mirror_thrash:waiting for 6.426474037362869 secs before reviving daemons 2026-02-22T22:05:25.322 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial026.stderr:2026-02-22T22:05:25.320+0000 7fd68f3a4640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 (PID: 76244) UID: 1000 2026-02-22T22:05:25.352 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6500 IOs, 2767966208 bytes 2026-02-22T22:05:25.388 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6600 IOs, 2780984320 bytes 2026-02-22T22:05:25.421 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6700 IOs, 2781601792 bytes 2026-02-22T22:05:25.469 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6800 IOs, 2823298048 bytes 2026-02-22T22:05:25.510 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6900 IOs, 2847981568 bytes 2026-02-22T22:05:25.561 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7000 IOs, 2900130816 bytes 2026-02-22T22:05:25.594 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7100 IOs, 2904339456 bytes 2026-02-22T22:05:25.653 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7200 IOs, 2970261504 bytes 2026-02-22T22:05:25.704 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7300 IOs, 3028919296 bytes 2026-02-22T22:05:25.750 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7400 IOs, 3054703616 bytes 2026-02-22T22:05:25.802 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7500 IOs, 3096476672 bytes 2026-02-22T22:05:25.840 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7600 IOs, 3111653376 bytes 2026-02-22T22:05:25.893 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7700 IOs, 3172156416 bytes 2026-02-22T22:05:25.938 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7800 IOs, 3214943232 bytes 2026-02-22T22:05:26.008 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7900 IOs, 3298457600 bytes 2026-02-22T22:05:26.049 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8000 IOs, 3315600384 bytes 2026-02-22T22:05:26.100 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8100 IOs, 3364551680 bytes 2026-02-22T22:05:26.149 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8200 IOs, 3410477056 bytes 2026-02-22T22:05:26.196 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8300 IOs, 3452104704 bytes 2026-02-22T22:05:26.254 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8400 IOs, 3516839936 bytes 2026-02-22T22:05:26.308 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8500 IOs, 3559446528 bytes 2026-02-22T22:05:26.351 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8600 IOs, 3592215552 bytes 2026-02-22T22:05:26.387 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8700 IOs, 3594982400 bytes 2026-02-22T22:05:26.437 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8800 IOs, 3649166336 bytes 2026-02-22T22:05:26.493 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8900 IOs, 3707567104 bytes 2026-02-22T22:05:26.538 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9000 IOs, 3736035328 bytes 2026-02-22T22:05:26.582 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9100 IOs, 3766726656 bytes 2026-02-22T22:05:26.620 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9200 IOs, 3801119744 bytes 2026-02-22T22:05:26.668 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9300 IOs, 3836249088 bytes 2026-02-22T22:05:26.716 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9400 IOs, 3895259136 bytes 2026-02-22T22:05:26.805 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9500 IOs, 4052777984 bytes 2026-02-22T22:05:26.866 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9600 IOs, 4106748928 bytes 2026-02-22T22:05:26.917 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9700 IOs, 4137801728 bytes 2026-02-22T22:05:26.963 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9800 IOs, 4179067904 bytes 2026-02-22T22:05:27.016 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9900 IOs, 4240567296 bytes 2026-02-22T22:05:27.086 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10000 IOs, 4328724480 bytes 2026-02-22T22:05:27.126 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10100 IOs, 4351286272 bytes 2026-02-22T22:05:27.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10200 IOs, 4374563840 bytes 2026-02-22T22:05:27.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10300 IOs, 4377858048 bytes 2026-02-22T22:05:27.261 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10400 IOs, 4422395904 bytes 2026-02-22T22:05:27.322 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10500 IOs, 4495300608 bytes 2026-02-22T22:05:27.369 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10600 IOs, 4527152128 bytes 2026-02-22T22:05:27.410 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10700 IOs, 4553992192 bytes 2026-02-22T22:05:27.450 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10800 IOs, 4576713728 bytes 2026-02-22T22:05:27.500 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10900 IOs, 4618243072 bytes 2026-02-22T22:05:27.534 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11000 IOs, 4624365568 bytes 2026-02-22T22:05:27.579 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11100 IOs, 4650818560 bytes 2026-02-22T22:05:27.630 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11200 IOs, 4701893632 bytes 2026-02-22T22:05:27.676 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11300 IOs, 4741203968 bytes 2026-02-22T22:05:27.709 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11400 IOs, 4754661376 bytes 2026-02-22T22:05:27.774 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11500 IOs, 4833576960 bytes 2026-02-22T22:05:27.828 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11600 IOs, 4880700416 bytes 2026-02-22T22:05:27.870 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11700 IOs, 4905337856 bytes 2026-02-22T22:05:27.930 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11800 IOs, 4981114880 bytes 2026-02-22T22:05:27.980 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11900 IOs, 5020151808 bytes 2026-02-22T22:05:28.029 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12000 IOs, 5068764160 bytes 2026-02-22T22:05:28.080 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12100 IOs, 5123069952 bytes 2026-02-22T22:05:28.145 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12200 IOs, 5177084928 bytes 2026-02-22T22:05:28.184 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12300 IOs, 5193503744 bytes 2026-02-22T22:05:28.228 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12400 IOs, 5222844416 bytes 2026-02-22T22:05:28.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12500 IOs, 5251176448 bytes 2026-02-22T22:05:28.308 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12600 IOs, 5259665408 bytes 2026-02-22T22:05:28.372 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12700 IOs, 5333167104 bytes 2026-02-22T22:05:28.413 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12800 IOs, 5354525696 bytes 2026-02-22T22:05:28.458 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12900 IOs, 5385637888 bytes 2026-02-22T22:05:28.510 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13000 IOs, 5442494464 bytes 2026-02-22T22:05:28.583 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13100 IOs, 5541882880 bytes 2026-02-22T22:05:28.620 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13200 IOs, 5554440192 bytes 2026-02-22T22:05:28.662 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13300 IOs, 5591493632 bytes 2026-02-22T22:05:28.703 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13400 IOs, 5611240448 bytes 2026-02-22T22:05:28.748 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13500 IOs, 5642043392 bytes 2026-02-22T22:05:28.778 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13600 IOs, 5642682368 bytes 2026-02-22T22:05:28.831 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13700 IOs, 5704007680 bytes 2026-02-22T22:05:28.870 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13800 IOs, 5716149248 bytes 2026-02-22T22:05:28.913 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13900 IOs, 5746563072 bytes 2026-02-22T22:05:28.949 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14000 IOs, 5759327232 bytes 2026-02-22T22:05:29.005 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14100 IOs, 5823170560 bytes 2026-02-22T22:05:29.058 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14200 IOs, 5874851840 bytes 2026-02-22T22:05:29.262 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ error_code=124 2026-02-22T22:05:29.262 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set -e 2026-02-22T22:05:29.263 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' 124 -eq 124 ']' 2026-02-22T22:05:29.263 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:05:29.263 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_present cluster1 mirror image_1 present 2026-02-22T22:05:29.263 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:29.263 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:29.263 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_1 2026-02-22T22:05:29.263 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=present 2026-02-22T22:05:29.263 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:05:29.263 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:05:29.263 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:05:29.263 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_1 2026-02-22T22:05:29.264 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:05:29.264 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:05:29.264 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_1 2026-02-22T22:05:29.264 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_1 2026-02-22T22:05:29.264 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:05:29.290 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image_id=12606f5f2552 2026-02-22T22:05:29.291 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:05:29.291 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.1 2026-02-22T22:05:29.392 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_present cluster1 mirror image_1 present 12606f5f2552 2026-02-22T22:05:29.392 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:29.392 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:29.392 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_1 2026-02-22T22:05:29.392 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=present 2026-02-22T22:05:29.393 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id=12606f5f2552 2026-02-22T22:05:29.393 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=deleted 2026-02-22T22:05:29.393 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_image_id 2026-02-22T22:05:29.393 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_1 2026-02-22T22:05:29.393 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:05:29.393 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:05:29.393 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_1 2026-02-22T22:05:29.393 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_1 2026-02-22T22:05:29.393 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:05:29.419 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_image_id=12606f5f2552 2026-02-22T22:05:29.420 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n 12606f5f2552 2026-02-22T22:05:29.420 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -z 12606f5f2552 -o 12606f5f2552 = 12606f5f2552 2026-02-22T22:05:29.420 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_state=present 2026-02-22T22:05:29.420 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test present = present 2026-02-22T22:05:29.420 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:05:29.420 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ disable_mirror cluster2 mirror image_1 2026-02-22T22:05:29.420 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:05:29.420 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:29.420 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_1 2026-02-22T22:05:29.420 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_1 2026-02-22T22:05:30.222 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial026.stderr:2026-02-22T22:05:30.221+0000 7f1baee3d640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x564723f75000 scan_remote_mirror_snapshots: failed to locate remote start snapshot: snap_id=185 2026-02-22T22:05:30.222 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial026.stderr:2026-02-22T22:05:30.221+0000 7f1baee3d640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x564723f75000 scan_remote_mirror_snapshots: split-brain detected: failed to find matching non-primary snapshot in remote image: local_snap_id_start=185, local_snap_ns=[mirror state=non-primary, complete=1, mirror_peer_uuids=, primary_mirror_uuid=cfa81703-817a-4074-adbc-5e6e200790bf, primary_snap_id=b9, last_copied_object_number=128, snap_seqs={185=18446744073709551614}] 2026-02-22T22:05:31.748 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.1 2026-02-22T22:05:31.749 INFO:teuthology.orchestra.run:waiting for 600 2026-02-22T22:05:31.749 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.1 2026-02-22T22:05:31.749 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Restarting daemon 2026-02-22T22:05:31.749 DEBUG:teuthology.orchestra.run.trial026:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 2026-02-22T22:05:31.752 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Started 2026-02-22T22:05:31.752 INFO:tasks.rbd_mirror_thrash:waiting for 102.86621414012566 secs before thrashing 2026-02-22T22:05:32.246 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.9659338252669594) > weight (0.25) 2026-02-22T22:05:32.246 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.4829454848196598) > weight (0.25) 2026-02-22T22:05:32.246 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.6342508554047648) > weight (0.25) 2026-02-22T22:05:32.247 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.3 2026-02-22T22:05:32.247 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Sent signal 15 2026-02-22T22:05:32.247 INFO:tasks.rbd_mirror_thrash:waiting for 5.454705036123119 secs before reviving daemons 2026-02-22T22:05:32.248 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial026.stderr:2026-02-22T22:05:32.246+0000 7fdba8e21640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.3 (PID: 62940) UID: 1000 2026-02-22T22:05:35.225 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Mirroring disabled 2026-02-22T22:05:35.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_present cluster1 mirror image_1 deleted 2026-02-22T22:05:35.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:35.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:35.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_1 2026-02-22T22:05:35.233 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=deleted 2026-02-22T22:05:35.233 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:05:35.233 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:05:35.233 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:05:35.233 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_1 2026-02-22T22:05:35.233 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:05:35.234 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:05:35.234 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_1 2026-02-22T22:05:35.234 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_1 2026-02-22T22:05:35.234 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:05:35.237 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial026.stderr:2026-02-22T22:05:35.235+0000 7efd821ce640 -1 rbd::mirror::ImageMap: 0x55d0798c9d40 handle_peer_ack_remove: failed to remove global_image_id=97b35fb1-1ed0-4b52-beb9-234ed156846b 2026-02-22T22:05:35.255 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_1: (2) No such file or directory 2026-02-22T22:05:35.257 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image_id= 2026-02-22T22:05:35.257 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:05:35.257 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.1 2026-02-22T22:05:35.358 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_present cluster1 mirror image_1 deleted '' 2026-02-22T22:05:35.358 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:05:35.358 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:35.358 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_1 2026-02-22T22:05:35.358 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=deleted 2026-02-22T22:05:35.358 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:05:35.358 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=deleted 2026-02-22T22:05:35.358 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_image_id 2026-02-22T22:05:35.358 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_1 2026-02-22T22:05:35.358 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:05:35.358 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:05:35.358 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_1 2026-02-22T22:05:35.358 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_1 2026-02-22T22:05:35.358 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:05:35.380 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_1: (2) No such file or directory 2026-02-22T22:05:35.382 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_image_id= 2026-02-22T22:05:35.382 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:05:35.382 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test deleted = deleted 2026-02-22T22:05:35.382 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:05:35.382 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ purge_snapshots cluster2 mirror image_1 2026-02-22T22:05:35.382 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:05:35.382 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:35.382 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_1 2026-02-22T22:05:35.382 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap purge mirror/image_1 2026-02-22T22:05:35.410 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image_retry cluster2 mirror image_1 2026-02-22T22:05:35.410 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:05:35.410 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:35.411 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_1 2026-02-22T22:05:35.411 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:05:35.411 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0 2026-02-22T22:05:35.411 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image cluster2 mirror image_1 2026-02-22T22:05:35.411 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:05:35.411 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:35.411 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_1 2026-02-22T22:05:35.411 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_1 2026-02-22T22:05:35.438 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 rm mirror/image_1 2026-02-22T22:05:37.703 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.3 2026-02-22T22:05:37.703 INFO:teuthology.orchestra.run:waiting for 600 2026-02-22T22:05:37.703 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.3 2026-02-22T22:05:37.703 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Restarting daemon 2026-02-22T22:05:37.703 DEBUG:teuthology.orchestra.run.trial026:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.3 2026-02-22T22:05:37.705 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Started 2026-02-22T22:05:37.705 INFO:tasks.rbd_mirror_thrash:waiting for 73.15446148240986 secs before thrashing 2026-02-22T22:05:58.776 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2026-02-22T22:05:58.780 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:05:58.780 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 10` 2026-02-22T22:05:58.780 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_2 2026-02-22T22:05:58.780 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image cluster2 mirror image_2 512M 2026-02-22T22:05:58.780 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:05:58.780 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T22:05:58.780 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:58.780 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T22:05:58.781 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_2 2026-02-22T22:05:58.781 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T22:05:58.781 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local size=128 2026-02-22T22:05:58.781 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n 512M ']' 2026-02-22T22:05:58.781 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ size=512M 2026-02-22T22:05:58.781 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T22:05:58.781 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering,exclusive-lock mirror/image_2 2026-02-22T22:05:58.813 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ enable_mirror cluster2 mirror image_2 2026-02-22T22:05:58.813 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:05:58.813 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:58.813 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_2 2026-02-22T22:05:58.813 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T22:05:58.813 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_2 snapshot 2026-02-22T22:05:59.689 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Mirroring enabled 2026-02-22T22:05:59.694 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 info mirror/image_2 2026-02-22T22:05:59.719 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_2': 2026-02-22T22:05:59.720 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 512 MiB in 128 objects 2026-02-22T22:05:59.720 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T22:05:59.720 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 1 2026-02-22T22:05:59.720 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 18ecd18c7e64 2026-02-22T22:05:59.720 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.18ecd18c7e64 2026-02-22T22:05:59.720 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T22:05:59.720 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock 2026-02-22T22:05:59.720 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T22:05:59.720 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T22:05:59.720 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 22:05:58 2026 2026-02-22T22:05:59.721 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 22:05:58 2026 2026-02-22T22:05:59.721 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 22:05:58 2026 2026-02-22T22:05:59.721 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T22:05:59.721 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T22:05:59.721 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: c3fdb820-6494-4fe9-ab00-51b2bab785a1 2026-02-22T22:05:59.721 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: true 2026-02-22T22:05:59.722 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ stress_write_image cluster2 mirror image_2 2026-02-22T22:05:59.722 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:05:59.722 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:05:59.722 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_2 2026-02-22T22:05:59.722 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-22T22:05:59.723 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local duration=11 2026-02-22T22:05:59.723 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set +e 2026-02-22T22:05:59.723 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ timeout 11s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_2 --debug-rbd=20 --debug-journaler=20 2026-02-22T22:05:59.792 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:100 IOs, 10328064 bytes 2026-02-22T22:05:59.851 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:200 IOs, 94932992 bytes 2026-02-22T22:05:59.906 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:300 IOs, 143909888 bytes 2026-02-22T22:05:59.955 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:400 IOs, 203037696 bytes 2026-02-22T22:06:00.016 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:500 IOs, 276758528 bytes 2026-02-22T22:06:00.053 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:600 IOs, 298916864 bytes 2026-02-22T22:06:00.100 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:700 IOs, 344502272 bytes 2026-02-22T22:06:00.133 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:800 IOs, 351083520 bytes 2026-02-22T22:06:00.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:900 IOs, 390593536 bytes 2026-02-22T22:06:00.229 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1000 IOs, 443990016 bytes 2026-02-22T22:06:00.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1100 IOs, 506761216 bytes 2026-02-22T22:06:00.341 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1200 IOs, 581933056 bytes 2026-02-22T22:06:00.380 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1300 IOs, 619322368 bytes 2026-02-22T22:06:00.430 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1400 IOs, 664696832 bytes 2026-02-22T22:06:00.472 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1500 IOs, 704291840 bytes 2026-02-22T22:06:00.514 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1600 IOs, 726099968 bytes 2026-02-22T22:06:00.550 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1700 IOs, 749619200 bytes 2026-02-22T22:06:00.599 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1800 IOs, 795561984 bytes 2026-02-22T22:06:00.632 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1900 IOs, 796196864 bytes 2026-02-22T22:06:00.689 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2000 IOs, 866285568 bytes 2026-02-22T22:06:00.723 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2100 IOs, 866910208 bytes 2026-02-22T22:06:00.768 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2200 IOs, 906718208 bytes 2026-02-22T22:06:00.818 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2300 IOs, 977687552 bytes 2026-02-22T22:06:00.881 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2400 IOs, 1055987712 bytes 2026-02-22T22:06:00.930 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2500 IOs, 1103719424 bytes 2026-02-22T22:06:00.981 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2600 IOs, 1158812672 bytes 2026-02-22T22:06:01.026 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2700 IOs, 1197724672 bytes 2026-02-22T22:06:01.070 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2800 IOs, 1236631552 bytes 2026-02-22T22:06:01.109 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2900 IOs, 1255389184 bytes 2026-02-22T22:06:01.141 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3000 IOs, 1263575040 bytes 2026-02-22T22:06:01.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3100 IOs, 1340116992 bytes 2026-02-22T22:06:01.233 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3200 IOs, 1340721152 bytes 2026-02-22T22:06:01.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3300 IOs, 1363690496 bytes 2026-02-22T22:06:01.307 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3400 IOs, 1375294464 bytes 2026-02-22T22:06:01.352 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3500 IOs, 1438253056 bytes 2026-02-22T22:06:01.397 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3600 IOs, 1446409216 bytes 2026-02-22T22:06:01.427 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3700 IOs, 1450092544 bytes 2026-02-22T22:06:01.468 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3800 IOs, 1474262016 bytes 2026-02-22T22:06:01.525 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3900 IOs, 1546936320 bytes 2026-02-22T22:06:01.582 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4000 IOs, 1606632448 bytes 2026-02-22T22:06:01.626 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4100 IOs, 1627459584 bytes 2026-02-22T22:06:01.682 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4200 IOs, 1689802752 bytes 2026-02-22T22:06:01.736 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4300 IOs, 1754519552 bytes 2026-02-22T22:06:01.788 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4400 IOs, 1796654080 bytes 2026-02-22T22:06:01.833 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4500 IOs, 1828424704 bytes 2026-02-22T22:06:01.904 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4600 IOs, 1924127744 bytes 2026-02-22T22:06:01.938 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4700 IOs, 1931080704 bytes 2026-02-22T22:06:01.968 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4800 IOs, 1931688960 bytes 2026-02-22T22:06:02.005 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4900 IOs, 1949132800 bytes 2026-02-22T22:06:02.065 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5000 IOs, 2018866176 bytes 2026-02-22T22:06:02.114 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5100 IOs, 2061568000 bytes 2026-02-22T22:06:02.179 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5200 IOs, 2129264640 bytes 2026-02-22T22:06:02.238 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5300 IOs, 2193406976 bytes 2026-02-22T22:06:02.301 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5400 IOs, 2273434624 bytes 2026-02-22T22:06:02.345 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5500 IOs, 2330321920 bytes 2026-02-22T22:06:02.393 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5600 IOs, 2353064960 bytes 2026-02-22T22:06:02.455 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5700 IOs, 2431200256 bytes 2026-02-22T22:06:02.491 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5800 IOs, 2444100608 bytes 2026-02-22T22:06:02.531 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5900 IOs, 2473822208 bytes 2026-02-22T22:06:02.587 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6000 IOs, 2530038784 bytes 2026-02-22T22:06:02.619 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6100 IOs, 2536881152 bytes 2026-02-22T22:06:02.663 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6200 IOs, 2576443392 bytes 2026-02-22T22:06:02.711 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6300 IOs, 2616955904 bytes 2026-02-22T22:06:02.743 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6400 IOs, 2617555968 bytes 2026-02-22T22:06:02.803 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6500 IOs, 2693838848 bytes 2026-02-22T22:06:02.840 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6600 IOs, 2704761856 bytes 2026-02-22T22:06:02.872 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6700 IOs, 2705387520 bytes 2026-02-22T22:06:02.916 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6800 IOs, 2734089216 bytes 2026-02-22T22:06:02.951 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6900 IOs, 2747888640 bytes 2026-02-22T22:06:03.019 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7000 IOs, 2836043776 bytes 2026-02-22T22:06:03.059 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7100 IOs, 2864574464 bytes 2026-02-22T22:06:03.103 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7200 IOs, 2894826496 bytes 2026-02-22T22:06:03.134 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7300 IOs, 2895427584 bytes 2026-02-22T22:06:03.174 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7400 IOs, 2918940672 bytes 2026-02-22T22:06:03.214 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7500 IOs, 2959812608 bytes 2026-02-22T22:06:03.248 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7600 IOs, 2970068992 bytes 2026-02-22T22:06:03.303 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7700 IOs, 3024601088 bytes 2026-02-22T22:06:03.334 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7800 IOs, 3025183744 bytes 2026-02-22T22:06:03.386 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7900 IOs, 3076248576 bytes 2026-02-22T22:06:03.436 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8000 IOs, 3129533440 bytes 2026-02-22T22:06:03.478 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8100 IOs, 3157307392 bytes 2026-02-22T22:06:03.542 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8200 IOs, 3273494528 bytes 2026-02-22T22:06:03.595 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8300 IOs, 3284473856 bytes 2026-02-22T22:06:03.638 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8400 IOs, 3307516928 bytes 2026-02-22T22:06:03.700 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8500 IOs, 3410693120 bytes 2026-02-22T22:06:03.765 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8600 IOs, 3467630592 bytes 2026-02-22T22:06:03.800 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8700 IOs, 3473453056 bytes 2026-02-22T22:06:03.852 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8800 IOs, 3523341312 bytes 2026-02-22T22:06:03.895 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8900 IOs, 3548693504 bytes 2026-02-22T22:06:03.932 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9000 IOs, 3564945408 bytes 2026-02-22T22:06:03.962 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9100 IOs, 3565585408 bytes 2026-02-22T22:06:04.005 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9200 IOs, 3608259584 bytes 2026-02-22T22:06:04.048 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9300 IOs, 3618468864 bytes 2026-02-22T22:06:04.103 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9400 IOs, 3674336256 bytes 2026-02-22T22:06:04.151 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9500 IOs, 3710945280 bytes 2026-02-22T22:06:04.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9600 IOs, 3764377600 bytes 2026-02-22T22:06:04.252 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9700 IOs, 3791083520 bytes 2026-02-22T22:06:04.301 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9800 IOs, 3833136128 bytes 2026-02-22T22:06:04.334 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9900 IOs, 3846939648 bytes 2026-02-22T22:06:04.370 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10000 IOs, 3854347264 bytes 2026-02-22T22:06:04.416 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10100 IOs, 3888479232 bytes 2026-02-22T22:06:04.466 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10200 IOs, 3933778944 bytes 2026-02-22T22:06:04.510 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10300 IOs, 3969745920 bytes 2026-02-22T22:06:04.546 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10400 IOs, 3986676736 bytes 2026-02-22T22:06:04.580 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10500 IOs, 4003877888 bytes 2026-02-22T22:06:04.618 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10600 IOs, 4013167616 bytes 2026-02-22T22:06:04.662 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10700 IOs, 4042201088 bytes 2026-02-22T22:06:04.712 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10800 IOs, 4077182976 bytes 2026-02-22T22:06:04.755 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10900 IOs, 4092876800 bytes 2026-02-22T22:06:04.796 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11000 IOs, 4121830400 bytes 2026-02-22T22:06:04.894 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11100 IOs, 4264763392 bytes 2026-02-22T22:06:04.930 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11200 IOs, 4265372672 bytes 2026-02-22T22:06:04.981 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11300 IOs, 4318717952 bytes 2026-02-22T22:06:05.041 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11400 IOs, 4380224512 bytes 2026-02-22T22:06:05.097 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11500 IOs, 4434480128 bytes 2026-02-22T22:06:05.164 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11600 IOs, 4522047488 bytes 2026-02-22T22:06:05.218 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11700 IOs, 4577024000 bytes 2026-02-22T22:06:05.283 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11800 IOs, 4632936448 bytes 2026-02-22T22:06:05.318 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11900 IOs, 4633547776 bytes 2026-02-22T22:06:05.380 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12000 IOs, 4706992128 bytes 2026-02-22T22:06:05.440 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12100 IOs, 4770997248 bytes 2026-02-22T22:06:05.490 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12200 IOs, 4813562880 bytes 2026-02-22T22:06:05.542 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12300 IOs, 4861833216 bytes 2026-02-22T22:06:05.602 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12400 IOs, 4926837760 bytes 2026-02-22T22:06:05.641 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12500 IOs, 4942155776 bytes 2026-02-22T22:06:05.684 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12600 IOs, 4966656000 bytes 2026-02-22T22:06:05.719 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12700 IOs, 4972020736 bytes 2026-02-22T22:06:05.757 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12800 IOs, 4983276544 bytes 2026-02-22T22:06:05.804 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12900 IOs, 5010710528 bytes 2026-02-22T22:06:06.051 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13000 IOs, 5061129216 bytes 2026-02-22T22:06:06.101 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13100 IOs, 5091926016 bytes 2026-02-22T22:06:06.169 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13200 IOs, 5173336064 bytes 2026-02-22T22:06:06.214 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13300 IOs, 5201854464 bytes 2026-02-22T22:06:06.264 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13400 IOs, 5241180160 bytes 2026-02-22T22:06:06.300 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13500 IOs, 5247665152 bytes 2026-02-22T22:06:06.346 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13600 IOs, 5283167232 bytes 2026-02-22T22:06:06.384 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13700 IOs, 5283769344 bytes 2026-02-22T22:06:06.426 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13800 IOs, 5332037632 bytes 2026-02-22T22:06:06.490 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13900 IOs, 5399384064 bytes 2026-02-22T22:06:06.548 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14000 IOs, 5447358464 bytes 2026-02-22T22:06:06.602 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14100 IOs, 5496192000 bytes 2026-02-22T22:06:06.643 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14200 IOs, 5508841472 bytes 2026-02-22T22:06:06.676 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14300 IOs, 5509452800 bytes 2026-02-22T22:06:06.731 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14400 IOs, 5556598784 bytes 2026-02-22T22:06:06.767 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14500 IOs, 5566443520 bytes 2026-02-22T22:06:06.807 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14600 IOs, 5583601664 bytes 2026-02-22T22:06:06.839 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14700 IOs, 5584214016 bytes 2026-02-22T22:06:06.879 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14800 IOs, 5603894272 bytes 2026-02-22T22:06:06.915 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14900 IOs, 5621743616 bytes 2026-02-22T22:06:06.958 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15000 IOs, 5640800256 bytes 2026-02-22T22:06:06.992 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15100 IOs, 5648608256 bytes 2026-02-22T22:06:07.034 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15200 IOs, 5666688000 bytes 2026-02-22T22:06:07.085 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15300 IOs, 5708752896 bytes 2026-02-22T22:06:07.140 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15400 IOs, 5760237568 bytes 2026-02-22T22:06:07.206 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15500 IOs, 5833393152 bytes 2026-02-22T22:06:07.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15600 IOs, 5909137408 bytes 2026-02-22T22:06:07.323 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15700 IOs, 5951500288 bytes 2026-02-22T22:06:07.376 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15800 IOs, 5997075456 bytes 2026-02-22T22:06:07.429 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15900 IOs, 6038338560 bytes 2026-02-22T22:06:07.500 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16000 IOs, 6126102528 bytes 2026-02-22T22:06:07.549 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16100 IOs, 6169768960 bytes 2026-02-22T22:06:07.614 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16200 IOs, 6236256256 bytes 2026-02-22T22:06:07.667 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16300 IOs, 6278344704 bytes 2026-02-22T22:06:07.754 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16400 IOs, 6401754112 bytes 2026-02-22T22:06:07.807 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16500 IOs, 6446031872 bytes 2026-02-22T22:06:07.874 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16600 IOs, 6533311488 bytes 2026-02-22T22:06:07.917 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16700 IOs, 6548013056 bytes 2026-02-22T22:06:07.974 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16800 IOs, 6605421568 bytes 2026-02-22T22:06:08.021 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16900 IOs, 6651262976 bytes 2026-02-22T22:06:08.090 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17000 IOs, 6743486464 bytes 2026-02-22T22:06:08.157 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17100 IOs, 6826292224 bytes 2026-02-22T22:06:08.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17200 IOs, 6849210368 bytes 2026-02-22T22:06:08.244 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17300 IOs, 6868488192 bytes 2026-02-22T22:06:08.289 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17400 IOs, 6900467712 bytes 2026-02-22T22:06:08.350 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17500 IOs, 6962571264 bytes 2026-02-22T22:06:08.414 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17600 IOs, 7027734528 bytes 2026-02-22T22:06:08.475 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17700 IOs, 7100987392 bytes 2026-02-22T22:06:08.538 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17800 IOs, 7176765440 bytes 2026-02-22T22:06:08.586 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17900 IOs, 7201676288 bytes 2026-02-22T22:06:08.638 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18000 IOs, 7232956416 bytes 2026-02-22T22:06:08.685 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18100 IOs, 7258548224 bytes 2026-02-22T22:06:08.719 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18200 IOs, 7260019712 bytes 2026-02-22T22:06:08.770 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18300 IOs, 7302728704 bytes 2026-02-22T22:06:08.815 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18400 IOs, 7326548992 bytes 2026-02-22T22:06:08.852 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18500 IOs, 7347625984 bytes 2026-02-22T22:06:08.908 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18600 IOs, 7415585792 bytes 2026-02-22T22:06:08.946 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18700 IOs, 7418889216 bytes 2026-02-22T22:06:09.000 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18800 IOs, 7463695360 bytes 2026-02-22T22:06:09.060 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18900 IOs, 7537933312 bytes 2026-02-22T22:06:09.116 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19000 IOs, 7589531648 bytes 2026-02-22T22:06:09.174 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19100 IOs, 7655284736 bytes 2026-02-22T22:06:09.223 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19200 IOs, 7692455936 bytes 2026-02-22T22:06:09.260 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19300 IOs, 7696150528 bytes 2026-02-22T22:06:09.306 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19400 IOs, 7726327808 bytes 2026-02-22T22:06:09.353 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19500 IOs, 7763916800 bytes 2026-02-22T22:06:09.398 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19600 IOs, 7791407104 bytes 2026-02-22T22:06:09.461 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19700 IOs, 7876759552 bytes 2026-02-22T22:06:09.519 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19800 IOs, 7940715520 bytes 2026-02-22T22:06:09.584 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19900 IOs, 8015166464 bytes 2026-02-22T22:06:09.629 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20000 IOs, 8046134272 bytes 2026-02-22T22:06:09.686 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20100 IOs, 8092145664 bytes 2026-02-22T22:06:09.740 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20200 IOs, 8149475328 bytes 2026-02-22T22:06:09.782 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20300 IOs, 8173271040 bytes 2026-02-22T22:06:09.835 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20400 IOs, 8220768256 bytes 2026-02-22T22:06:09.889 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20500 IOs, 8278051840 bytes 2026-02-22T22:06:09.965 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20600 IOs, 8376568832 bytes 2026-02-22T22:06:10.008 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20700 IOs, 8405286912 bytes 2026-02-22T22:06:10.055 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20800 IOs, 8446464000 bytes 2026-02-22T22:06:10.101 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20900 IOs, 8466793472 bytes 2026-02-22T22:06:10.147 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21000 IOs, 8488828928 bytes 2026-02-22T22:06:10.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21100 IOs, 8545468416 bytes 2026-02-22T22:06:10.252 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21200 IOs, 8583611392 bytes 2026-02-22T22:06:10.290 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21300 IOs, 8600044544 bytes 2026-02-22T22:06:10.333 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21400 IOs, 8627084288 bytes 2026-02-22T22:06:10.381 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21500 IOs, 8660941824 bytes 2026-02-22T22:06:10.451 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21600 IOs, 8736929792 bytes 2026-02-22T22:06:10.521 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21700 IOs, 8833964032 bytes 2026-02-22T22:06:10.574 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21800 IOs, 8900287488 bytes 2026-02-22T22:06:10.621 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21900 IOs, 8922296320 bytes 2026-02-22T22:06:10.671 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22000 IOs, 8965090304 bytes 2026-02-22T22:06:10.715 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22100 IOs, 8975610880 bytes 2026-02-22T22:06:10.729 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ error_code=124 2026-02-22T22:06:10.729 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set -e 2026-02-22T22:06:10.729 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' 124 -eq 124 ']' 2026-02-22T22:06:10.730 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:06:10.730 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_present cluster1 mirror image_2 present 2026-02-22T22:06:10.730 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:06:10.730 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:06:10.730 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_2 2026-02-22T22:06:10.730 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=present 2026-02-22T22:06:10.730 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:06:10.730 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:06:10.730 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:06:10.730 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_2 2026-02-22T22:06:10.730 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:06:10.731 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:06:10.731 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_2 2026-02-22T22:06:10.731 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_2 2026-02-22T22:06:10.731 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:06:10.757 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image_id=12605eba9325 2026-02-22T22:06:10.757 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:06:10.758 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.1 2026-02-22T22:06:10.859 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_present cluster1 mirror image_2 present 12605eba9325 2026-02-22T22:06:10.859 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:06:10.859 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:06:10.859 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_2 2026-02-22T22:06:10.859 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=present 2026-02-22T22:06:10.860 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id=12605eba9325 2026-02-22T22:06:10.860 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=deleted 2026-02-22T22:06:10.860 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_image_id 2026-02-22T22:06:10.860 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_2 2026-02-22T22:06:10.860 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:06:10.860 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:06:10.860 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_2 2026-02-22T22:06:10.860 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_2 2026-02-22T22:06:10.860 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:06:10.886 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_image_id=12605eba9325 2026-02-22T22:06:10.886 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n 12605eba9325 2026-02-22T22:06:10.886 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -z 12605eba9325 -o 12605eba9325 = 12605eba9325 2026-02-22T22:06:10.886 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_state=present 2026-02-22T22:06:10.886 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test present = present 2026-02-22T22:06:10.886 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:06:10.886 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ disable_mirror cluster2 mirror image_2 2026-02-22T22:06:10.886 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:06:10.886 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:06:10.887 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_2 2026-02-22T22:06:10.887 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_2 2026-02-22T22:06:10.981 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial026.stderr:2026-02-22T22:06:10.980+0000 7f1baf63e640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x564724962c00 scan_remote_mirror_snapshots: failed to locate remote start snapshot: snap_id=186 2026-02-22T22:06:10.982 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial026.stderr:2026-02-22T22:06:10.980+0000 7f1baf63e640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x564724962c00 scan_remote_mirror_snapshots: split-brain detected: failed to find matching non-primary snapshot in remote image: local_snap_id_start=186, local_snap_ns=[mirror state=non-primary, complete=1, mirror_peer_uuids=, primary_mirror_uuid=cfa81703-817a-4074-adbc-5e6e200790bf, primary_snap_id=ba, last_copied_object_number=128, snap_seqs={186=18446744073709551614}] 2026-02-22T22:06:14.739 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial026.stderr:2026-02-22T22:06:14.737+0000 7f4b1f3b1640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x562edb428b60 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-22T22:06:14.739 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial026.stderr:2026-02-22T22:06:14.737+0000 7f4b263bf640 -1 rbd::mirror::ImageReplayer: 0x562edb6e0c80 [3/c3fdb820-6494-4fe9-ab00-51b2bab785a1] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-22T22:06:15.982 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Mirroring disabled 2026-02-22T22:06:15.987 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_present cluster1 mirror image_2 deleted 2026-02-22T22:06:15.987 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:06:15.987 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:06:15.987 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_2 2026-02-22T22:06:15.987 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=deleted 2026-02-22T22:06:15.987 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:06:15.987 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:06:15.987 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:06:15.987 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_2 2026-02-22T22:06:15.988 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:06:15.988 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:06:15.988 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_2 2026-02-22T22:06:15.988 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_2 2026-02-22T22:06:15.988 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:06:16.013 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_2: (2) No such file or directory 2026-02-22T22:06:16.015 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image_id= 2026-02-22T22:06:16.015 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:06:16.015 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.1 2026-02-22T22:06:16.117 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_present cluster1 mirror image_2 deleted '' 2026-02-22T22:06:16.118 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:06:16.118 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:06:16.118 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_2 2026-02-22T22:06:16.118 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=deleted 2026-02-22T22:06:16.118 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:06:16.118 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=deleted 2026-02-22T22:06:16.118 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_image_id 2026-02-22T22:06:16.118 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_2 2026-02-22T22:06:16.119 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:06:16.119 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:06:16.119 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_2 2026-02-22T22:06:16.119 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_2 2026-02-22T22:06:16.119 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:06:16.139 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_2: (2) No such file or directory 2026-02-22T22:06:16.142 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_image_id= 2026-02-22T22:06:16.142 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:06:16.142 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test deleted = deleted 2026-02-22T22:06:16.142 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:06:16.142 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ purge_snapshots cluster2 mirror image_2 2026-02-22T22:06:16.142 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:06:16.142 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:06:16.142 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_2 2026-02-22T22:06:16.142 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap purge mirror/image_2 2026-02-22T22:06:16.169 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image_retry cluster2 mirror image_2 2026-02-22T22:06:16.170 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:06:16.170 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:06:16.170 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_2 2026-02-22T22:06:16.170 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:06:16.170 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0 2026-02-22T22:06:16.171 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image cluster2 mirror image_2 2026-02-22T22:06:16.171 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:06:16.171 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:06:16.171 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_2 2026-02-22T22:06:16.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_2 2026-02-22T22:06:16.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 rm mirror/image_2 2026-02-22T22:06:41.327 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2026-02-22T22:06:41.331 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:06:41.331 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 10` 2026-02-22T22:06:41.331 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_3 2026-02-22T22:06:41.332 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image cluster2 mirror image_3 512M 2026-02-22T22:06:41.332 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:06:41.332 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T22:06:41.332 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:06:41.332 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T22:06:41.332 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_3 2026-02-22T22:06:41.332 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T22:06:41.332 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local size=128 2026-02-22T22:06:41.332 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n 512M ']' 2026-02-22T22:06:41.332 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ size=512M 2026-02-22T22:06:41.333 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T22:06:41.333 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering,exclusive-lock mirror/image_3 2026-02-22T22:06:41.363 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ enable_mirror cluster2 mirror image_3 2026-02-22T22:06:41.363 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:06:41.363 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:06:41.363 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_3 2026-02-22T22:06:41.364 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T22:06:41.364 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_3 snapshot 2026-02-22T22:06:42.235 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Mirroring enabled 2026-02-22T22:06:42.241 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 info mirror/image_3 2026-02-22T22:06:42.265 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_3': 2026-02-22T22:06:42.265 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 512 MiB in 128 objects 2026-02-22T22:06:42.265 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T22:06:42.265 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 1 2026-02-22T22:06:42.265 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 1900aa1683ca 2026-02-22T22:06:42.266 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.1900aa1683ca 2026-02-22T22:06:42.266 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T22:06:42.266 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock 2026-02-22T22:06:42.266 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T22:06:42.266 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T22:06:42.266 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 22:06:41 2026 2026-02-22T22:06:42.266 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 22:06:41 2026 2026-02-22T22:06:42.266 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 22:06:41 2026 2026-02-22T22:06:42.266 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T22:06:42.266 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T22:06:42.267 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: f4b8a9d0-4ba2-41b8-81aa-a79265141a66 2026-02-22T22:06:42.267 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: true 2026-02-22T22:06:42.267 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ stress_write_image cluster2 mirror image_3 2026-02-22T22:06:42.268 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:06:42.268 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:06:42.268 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_3 2026-02-22T22:06:42.268 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-22T22:06:42.269 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local duration=5 2026-02-22T22:06:42.269 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set +e 2026-02-22T22:06:42.269 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ timeout 5s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_3 --debug-rbd=20 --debug-journaler=20 2026-02-22T22:06:42.378 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:100 IOs, 101420032 bytes 2026-02-22T22:06:42.425 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:200 IOs, 145437696 bytes 2026-02-22T22:06:42.481 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:300 IOs, 215317504 bytes 2026-02-22T22:06:42.527 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:400 IOs, 255556608 bytes 2026-02-22T22:06:42.582 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:500 IOs, 323138560 bytes 2026-02-22T22:06:42.629 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:600 IOs, 359050240 bytes 2026-02-22T22:06:42.684 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:700 IOs, 441042944 bytes 2026-02-22T22:06:42.752 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:800 IOs, 507534336 bytes 2026-02-22T22:06:42.791 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:900 IOs, 526264320 bytes 2026-02-22T22:06:42.827 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1000 IOs, 540722176 bytes 2026-02-22T22:06:42.865 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1100 IOs, 569467904 bytes 2026-02-22T22:06:42.897 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1200 IOs, 589630464 bytes 2026-02-22T22:06:42.942 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1300 IOs, 614807552 bytes 2026-02-22T22:06:43.000 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1400 IOs, 685980672 bytes 2026-02-22T22:06:43.054 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1500 IOs, 743926784 bytes 2026-02-22T22:06:43.096 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1600 IOs, 790347776 bytes 2026-02-22T22:06:43.139 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1700 IOs, 815076352 bytes 2026-02-22T22:06:43.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1800 IOs, 856333312 bytes 2026-02-22T22:06:43.235 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1900 IOs, 880972800 bytes 2026-02-22T22:06:43.282 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2000 IOs, 925373440 bytes 2026-02-22T22:06:43.339 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2100 IOs, 992972800 bytes 2026-02-22T22:06:43.401 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2200 IOs, 1072407552 bytes 2026-02-22T22:06:43.458 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2300 IOs, 1140841472 bytes 2026-02-22T22:06:43.513 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2400 IOs, 1194671104 bytes 2026-02-22T22:06:43.549 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2500 IOs, 1209060352 bytes 2026-02-22T22:06:43.609 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2600 IOs, 1301708800 bytes 2026-02-22T22:06:43.661 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2700 IOs, 1328516096 bytes 2026-02-22T22:06:43.715 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2800 IOs, 1377031168 bytes 2026-02-22T22:06:43.764 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2900 IOs, 1416435712 bytes 2026-02-22T22:06:43.825 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3000 IOs, 1511720960 bytes 2026-02-22T22:06:43.877 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3100 IOs, 1554524160 bytes 2026-02-22T22:06:43.920 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3200 IOs, 1593903104 bytes 2026-02-22T22:06:43.954 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3300 IOs, 1598231552 bytes 2026-02-22T22:06:43.993 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3400 IOs, 1613252608 bytes 2026-02-22T22:06:44.039 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3500 IOs, 1660178432 bytes 2026-02-22T22:06:44.082 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3600 IOs, 1689638912 bytes 2026-02-22T22:06:44.135 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3700 IOs, 1752714240 bytes 2026-02-22T22:06:44.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3800 IOs, 1826147328 bytes 2026-02-22T22:06:44.247 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3900 IOs, 1873124352 bytes 2026-02-22T22:06:44.297 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4000 IOs, 1929419776 bytes 2026-02-22T22:06:44.357 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4100 IOs, 2002325504 bytes 2026-02-22T22:06:44.388 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4200 IOs, 2002944000 bytes 2026-02-22T22:06:44.425 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4300 IOs, 2023944192 bytes 2026-02-22T22:06:44.469 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4400 IOs, 2060022784 bytes 2026-02-22T22:06:44.516 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4500 IOs, 2103719936 bytes 2026-02-22T22:06:44.557 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4600 IOs, 2133613568 bytes 2026-02-22T22:06:44.622 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4700 IOs, 2221317120 bytes 2026-02-22T22:06:44.675 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4800 IOs, 2275606528 bytes 2026-02-22T22:06:44.712 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4900 IOs, 2278122496 bytes 2026-02-22T22:06:44.769 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5000 IOs, 2341411840 bytes 2026-02-22T22:06:44.804 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5100 IOs, 2362547200 bytes 2026-02-22T22:06:44.850 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5200 IOs, 2396261376 bytes 2026-02-22T22:06:44.905 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5300 IOs, 2445764608 bytes 2026-02-22T22:06:44.942 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5400 IOs, 2454865920 bytes 2026-02-22T22:06:44.973 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5500 IOs, 2462044160 bytes 2026-02-22T22:06:45.018 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5600 IOs, 2506419200 bytes 2026-02-22T22:06:45.068 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5700 IOs, 2534500352 bytes 2026-02-22T22:06:45.108 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5800 IOs, 2557907968 bytes 2026-02-22T22:06:45.145 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5900 IOs, 2587012096 bytes 2026-02-22T22:06:45.184 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6000 IOs, 2611549184 bytes 2026-02-22T22:06:45.223 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6100 IOs, 2638824448 bytes 2026-02-22T22:06:45.260 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6200 IOs, 2657545216 bytes 2026-02-22T22:06:45.292 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6300 IOs, 2667435008 bytes 2026-02-22T22:06:45.338 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6400 IOs, 2722813952 bytes 2026-02-22T22:06:45.387 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6500 IOs, 2758363136 bytes 2026-02-22T22:06:45.430 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6600 IOs, 2802354176 bytes 2026-02-22T22:06:45.496 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6700 IOs, 2876023808 bytes 2026-02-22T22:06:45.537 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6800 IOs, 2908654592 bytes 2026-02-22T22:06:45.575 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6900 IOs, 2929700864 bytes 2026-02-22T22:06:45.621 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7000 IOs, 2970172416 bytes 2026-02-22T22:06:45.659 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7100 IOs, 2986457088 bytes 2026-02-22T22:06:45.719 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7200 IOs, 3057253376 bytes 2026-02-22T22:06:45.783 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7300 IOs, 3123259392 bytes 2026-02-22T22:06:45.836 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7400 IOs, 3167497216 bytes 2026-02-22T22:06:45.874 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7500 IOs, 3194920960 bytes 2026-02-22T22:06:45.906 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7600 IOs, 3195548672 bytes 2026-02-22T22:06:45.941 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7700 IOs, 3216588800 bytes 2026-02-22T22:06:45.983 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7800 IOs, 3246447616 bytes 2026-02-22T22:06:46.036 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7900 IOs, 3303195648 bytes 2026-02-22T22:06:46.096 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8000 IOs, 3374275584 bytes 2026-02-22T22:06:46.135 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8100 IOs, 3411522560 bytes 2026-02-22T22:06:46.188 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8200 IOs, 3468640256 bytes 2026-02-22T22:06:46.249 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8300 IOs, 3528469504 bytes 2026-02-22T22:06:46.301 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8400 IOs, 3574673408 bytes 2026-02-22T22:06:46.360 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8500 IOs, 3634845696 bytes 2026-02-22T22:06:46.409 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8600 IOs, 3675319296 bytes 2026-02-22T22:06:46.453 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8700 IOs, 3707153408 bytes 2026-02-22T22:06:46.495 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8800 IOs, 3735685120 bytes 2026-02-22T22:06:46.559 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8900 IOs, 3820463104 bytes 2026-02-22T22:06:46.624 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9000 IOs, 3916658688 bytes 2026-02-22T22:06:46.677 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9100 IOs, 3954506752 bytes 2026-02-22T22:06:46.718 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9200 IOs, 3980982272 bytes 2026-02-22T22:06:46.764 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9300 IOs, 4041824256 bytes 2026-02-22T22:06:46.838 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9400 IOs, 4109099008 bytes 2026-02-22T22:06:46.872 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9500 IOs, 4117435392 bytes 2026-02-22T22:06:46.901 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9600 IOs, 4118079488 bytes 2026-02-22T22:06:46.932 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9700 IOs, 4123939840 bytes 2026-02-22T22:06:46.983 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9800 IOs, 4171098112 bytes 2026-02-22T22:06:47.054 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9900 IOs, 4272111616 bytes 2026-02-22T22:06:47.096 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10000 IOs, 4290822144 bytes 2026-02-22T22:06:47.137 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10100 IOs, 4315222016 bytes 2026-02-22T22:06:47.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10200 IOs, 4395505664 bytes 2026-02-22T22:06:47.234 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10300 IOs, 4409430016 bytes 2026-02-22T22:06:47.274 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ error_code=124 2026-02-22T22:06:47.274 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set -e 2026-02-22T22:06:47.274 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' 124 -eq 124 ']' 2026-02-22T22:06:47.274 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:06:47.274 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_present cluster1 mirror image_3 present 2026-02-22T22:06:47.274 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:06:47.274 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:06:47.274 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_3 2026-02-22T22:06:47.274 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=present 2026-02-22T22:06:47.275 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:06:47.275 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:06:47.275 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:06:47.275 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_3 2026-02-22T22:06:47.275 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:06:47.275 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:06:47.275 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_3 2026-02-22T22:06:47.275 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_3 2026-02-22T22:06:47.275 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:06:47.304 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image_id=1260979197ab 2026-02-22T22:06:47.304 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:06:47.304 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.1 2026-02-22T22:06:47.405 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_present cluster1 mirror image_3 present 1260979197ab 2026-02-22T22:06:47.406 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:06:47.406 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:06:47.406 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_3 2026-02-22T22:06:47.406 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=present 2026-02-22T22:06:47.406 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id=1260979197ab 2026-02-22T22:06:47.406 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=deleted 2026-02-22T22:06:47.406 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_image_id 2026-02-22T22:06:47.406 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_3 2026-02-22T22:06:47.406 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:06:47.406 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:06:47.407 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_3 2026-02-22T22:06:47.407 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_3 2026-02-22T22:06:47.407 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:06:47.433 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_image_id=1260979197ab 2026-02-22T22:06:47.433 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n 1260979197ab 2026-02-22T22:06:47.433 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -z 1260979197ab -o 1260979197ab = 1260979197ab 2026-02-22T22:06:47.433 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_state=present 2026-02-22T22:06:47.433 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test present = present 2026-02-22T22:06:47.433 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:06:47.433 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ disable_mirror cluster2 mirror image_3 2026-02-22T22:06:47.433 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:06:47.433 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:06:47.433 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_3 2026-02-22T22:06:47.434 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_3 2026-02-22T22:06:48.277 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial026.stderr:2026-02-22T22:06:48.275+0000 7f1baf63e640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x564724af8800 scan_remote_mirror_snapshots: failed to locate remote start snapshot: snap_id=187 2026-02-22T22:06:48.277 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial026.stderr:2026-02-22T22:06:48.275+0000 7f1baf63e640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x564724af8800 scan_remote_mirror_snapshots: split-brain detected: failed to find matching non-primary snapshot in remote image: local_snap_id_start=187, local_snap_ns=[mirror state=non-primary, complete=1, mirror_peer_uuids=, primary_mirror_uuid=cfa81703-817a-4074-adbc-5e6e200790bf, primary_snap_id=bb, last_copied_object_number=128, snap_seqs={187=18446744073709551614}] 2026-02-22T22:06:50.860 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.0 2026-02-22T22:06:50.861 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Sent signal 15 2026-02-22T22:06:50.861 INFO:tasks.rbd_mirror_thrash:waiting for 4.4138460794941325 secs before reviving daemons 2026-02-22T22:06:50.861 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial026.stderr:2026-02-22T22:06:50.860+0000 7f4b26bc0640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 (PID: 68916) UID: 1000 2026-02-22T22:06:53.279 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Mirroring disabled 2026-02-22T22:06:53.287 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_present cluster1 mirror image_3 deleted 2026-02-22T22:06:53.287 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:06:53.287 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:06:53.287 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_3 2026-02-22T22:06:53.287 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=deleted 2026-02-22T22:06:53.287 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:06:53.288 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:06:53.288 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:06:53.288 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_3 2026-02-22T22:06:53.288 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:06:53.288 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:06:53.288 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_3 2026-02-22T22:06:53.288 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_3 2026-02-22T22:06:53.288 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:06:53.292 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial026.stderr:2026-02-22T22:06:53.289+0000 7efd819cd640 -1 rbd::mirror::ImageMap: 0x55d0798c9d40 handle_peer_ack_remove: failed to remove global_image_id=f4b8a9d0-4ba2-41b8-81aa-a79265141a66 2026-02-22T22:06:53.311 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_3: (2) No such file or directory 2026-02-22T22:06:53.312 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image_id= 2026-02-22T22:06:53.312 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:06:53.313 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.1 2026-02-22T22:06:53.413 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_present cluster1 mirror image_3 deleted '' 2026-02-22T22:06:53.413 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:06:53.413 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:06:53.413 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_3 2026-02-22T22:06:53.413 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=deleted 2026-02-22T22:06:53.413 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:06:53.414 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=deleted 2026-02-22T22:06:53.414 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_image_id 2026-02-22T22:06:53.414 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_3 2026-02-22T22:06:53.414 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:06:53.414 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:06:53.414 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_3 2026-02-22T22:06:53.414 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_3 2026-02-22T22:06:53.414 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:06:53.438 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_3: (2) No such file or directory 2026-02-22T22:06:53.440 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_image_id= 2026-02-22T22:06:53.440 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:06:53.440 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test deleted = deleted 2026-02-22T22:06:53.440 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:06:53.440 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ purge_snapshots cluster2 mirror image_3 2026-02-22T22:06:53.440 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:06:53.440 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:06:53.440 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_3 2026-02-22T22:06:53.440 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap purge mirror/image_3 2026-02-22T22:06:53.469 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image_retry cluster2 mirror image_3 2026-02-22T22:06:53.469 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:06:53.469 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:06:53.469 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_3 2026-02-22T22:06:53.469 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:06:53.469 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0 2026-02-22T22:06:53.470 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image cluster2 mirror image_3 2026-02-22T22:06:53.470 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:06:53.470 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:06:53.470 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_3 2026-02-22T22:06:53.470 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_3 2026-02-22T22:06:53.496 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 rm mirror/image_3 2026-02-22T22:06:55.275 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.0 2026-02-22T22:06:55.276 INFO:teuthology.orchestra.run:waiting for 600 2026-02-22T22:06:55.276 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.0 2026-02-22T22:06:55.276 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Restarting daemon 2026-02-22T22:06:55.277 DEBUG:teuthology.orchestra.run.trial026:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 2026-02-22T22:06:55.279 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Started 2026-02-22T22:06:55.279 INFO:tasks.rbd_mirror_thrash:waiting for 89.72553317498051 secs before thrashing 2026-02-22T22:07:14.619 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.9472493373053255) > weight (0.25) 2026-02-22T22:07:14.620 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.1 2026-02-22T22:07:14.620 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Sent signal 15 2026-02-22T22:07:14.620 INFO:tasks.rbd_mirror_thrash:waiting for 2.7153630548970433 secs before reviving daemons 2026-02-22T22:07:14.621 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial026.stderr:2026-02-22T22:07:14.619+0000 7f9ebafbe640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 (PID: 80789) UID: 1000 2026-02-22T22:07:17.337 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.1 2026-02-22T22:07:17.337 INFO:teuthology.orchestra.run:waiting for 600 2026-02-22T22:07:17.337 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.1 2026-02-22T22:07:17.337 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Restarting daemon 2026-02-22T22:07:17.337 DEBUG:teuthology.orchestra.run.trial026:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 2026-02-22T22:07:17.339 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Started 2026-02-22T22:07:17.339 INFO:tasks.rbd_mirror_thrash:waiting for 101.75526517126202 secs before thrashing 2026-02-22T22:07:18.534 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2026-02-22T22:07:18.537 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:07:18.538 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 10` 2026-02-22T22:07:18.538 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_4 2026-02-22T22:07:18.538 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image cluster2 mirror image_4 512M 2026-02-22T22:07:18.538 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:07:18.538 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T22:07:18.538 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:07:18.538 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T22:07:18.538 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_4 2026-02-22T22:07:18.538 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T22:07:18.539 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local size=128 2026-02-22T22:07:18.539 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n 512M ']' 2026-02-22T22:07:18.539 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ size=512M 2026-02-22T22:07:18.539 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T22:07:18.539 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering,exclusive-lock mirror/image_4 2026-02-22T22:07:18.568 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ enable_mirror cluster2 mirror image_4 2026-02-22T22:07:18.569 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:07:18.569 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:07:18.569 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_4 2026-02-22T22:07:18.569 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T22:07:18.569 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_4 snapshot 2026-02-22T22:07:19.244 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Mirroring enabled 2026-02-22T22:07:19.249 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 info mirror/image_4 2026-02-22T22:07:19.273 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_4': 2026-02-22T22:07:19.273 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 512 MiB in 128 objects 2026-02-22T22:07:19.274 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T22:07:19.274 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 1 2026-02-22T22:07:19.274 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 1925b51ba510 2026-02-22T22:07:19.274 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.1925b51ba510 2026-02-22T22:07:19.274 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T22:07:19.274 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock 2026-02-22T22:07:19.274 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T22:07:19.274 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T22:07:19.274 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 22:07:18 2026 2026-02-22T22:07:19.274 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 22:07:18 2026 2026-02-22T22:07:19.275 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 22:07:18 2026 2026-02-22T22:07:19.275 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T22:07:19.275 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T22:07:19.275 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: 8d1fbdd5-5eb7-4292-96f3-e14211e745de 2026-02-22T22:07:19.275 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: true 2026-02-22T22:07:19.276 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ stress_write_image cluster2 mirror image_4 2026-02-22T22:07:19.276 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:07:19.276 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:07:19.276 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_4 2026-02-22T22:07:19.276 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-22T22:07:19.277 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local duration=14 2026-02-22T22:07:19.277 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set +e 2026-02-22T22:07:19.277 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ timeout 14s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_4 --debug-rbd=20 --debug-journaler=20 2026-02-22T22:07:19.349 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:100 IOs, 19663872 bytes 2026-02-22T22:07:19.391 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:200 IOs, 53261312 bytes 2026-02-22T22:07:19.436 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:300 IOs, 95289344 bytes 2026-02-22T22:07:19.478 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:400 IOs, 126334976 bytes 2026-02-22T22:07:19.511 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:500 IOs, 131748864 bytes 2026-02-22T22:07:19.562 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:600 IOs, 198360064 bytes 2026-02-22T22:07:19.607 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:700 IOs, 221628416 bytes 2026-02-22T22:07:19.650 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:800 IOs, 257950720 bytes 2026-02-22T22:07:19.682 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:900 IOs, 263936000 bytes 2026-02-22T22:07:19.723 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1000 IOs, 295291904 bytes 2026-02-22T22:07:19.766 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1100 IOs, 333739008 bytes 2026-02-22T22:07:19.795 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1200 IOs, 334360576 bytes 2026-02-22T22:07:19.830 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1300 IOs, 378995712 bytes 2026-02-22T22:07:19.904 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1400 IOs, 457596928 bytes 2026-02-22T22:07:19.949 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1500 IOs, 489077760 bytes 2026-02-22T22:07:20.001 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1600 IOs, 551899136 bytes 2026-02-22T22:07:20.061 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1700 IOs, 660049920 bytes 2026-02-22T22:07:20.116 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1800 IOs, 709809152 bytes 2026-02-22T22:07:20.147 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1900 IOs, 710411264 bytes 2026-02-22T22:07:20.185 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2000 IOs, 738228224 bytes 2026-02-22T22:07:20.236 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2100 IOs, 797552640 bytes 2026-02-22T22:07:20.282 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2200 IOs, 839519232 bytes 2026-02-22T22:07:20.329 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2300 IOs, 889539584 bytes 2026-02-22T22:07:20.356 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2400 IOs, 890164224 bytes 2026-02-22T22:07:20.402 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2500 IOs, 928346112 bytes 2026-02-22T22:07:20.443 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2600 IOs, 968262656 bytes 2026-02-22T22:07:20.480 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2700 IOs, 974488576 bytes 2026-02-22T22:07:20.528 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2800 IOs, 1030933504 bytes 2026-02-22T22:07:20.565 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2900 IOs, 1033853952 bytes 2026-02-22T22:07:20.615 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3000 IOs, 1088650240 bytes 2026-02-22T22:07:20.657 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3100 IOs, 1127437312 bytes 2026-02-22T22:07:20.701 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3200 IOs, 1149325312 bytes 2026-02-22T22:07:20.745 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3300 IOs, 1184722944 bytes 2026-02-22T22:07:20.798 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3400 IOs, 1241962496 bytes 2026-02-22T22:07:20.830 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3500 IOs, 1251252224 bytes 2026-02-22T22:07:20.863 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3600 IOs, 1251833856 bytes 2026-02-22T22:07:20.908 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3700 IOs, 1290522624 bytes 2026-02-22T22:07:20.947 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3800 IOs, 1312623616 bytes 2026-02-22T22:07:20.990 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3900 IOs, 1356230656 bytes 2026-02-22T22:07:21.030 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4000 IOs, 1380653056 bytes 2026-02-22T22:07:21.079 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4100 IOs, 1439649792 bytes 2026-02-22T22:07:21.136 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4200 IOs, 1498561536 bytes 2026-02-22T22:07:21.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4300 IOs, 1525789696 bytes 2026-02-22T22:07:21.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4400 IOs, 1546686464 bytes 2026-02-22T22:07:21.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4500 IOs, 1612979200 bytes 2026-02-22T22:07:21.316 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4600 IOs, 1648523264 bytes 2026-02-22T22:07:21.358 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4700 IOs, 1681518592 bytes 2026-02-22T22:07:21.405 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4800 IOs, 1720687616 bytes 2026-02-22T22:07:21.453 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4900 IOs, 1764417536 bytes 2026-02-22T22:07:21.489 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5000 IOs, 1790971904 bytes 2026-02-22T22:07:21.545 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5100 IOs, 1844581376 bytes 2026-02-22T22:07:21.591 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5200 IOs, 1889356800 bytes 2026-02-22T22:07:21.624 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5300 IOs, 1902819328 bytes 2026-02-22T22:07:21.654 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5400 IOs, 1906332672 bytes 2026-02-22T22:07:21.688 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5500 IOs, 1919462400 bytes 2026-02-22T22:07:21.740 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5600 IOs, 1976099840 bytes 2026-02-22T22:07:21.801 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5700 IOs, 2043920384 bytes 2026-02-22T22:07:21.853 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5800 IOs, 2098317312 bytes 2026-02-22T22:07:21.902 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5900 IOs, 2159670272 bytes 2026-02-22T22:07:21.955 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6000 IOs, 2201053184 bytes 2026-02-22T22:07:21.986 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6100 IOs, 2208019456 bytes 2026-02-22T22:07:22.023 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6200 IOs, 2220327936 bytes 2026-02-22T22:07:22.061 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6300 IOs, 2250517504 bytes 2026-02-22T22:07:22.114 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6400 IOs, 2293701632 bytes 2026-02-22T22:07:22.361 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6500 IOs, 2346084352 bytes 2026-02-22T22:07:22.403 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6600 IOs, 2360749056 bytes 2026-02-22T22:07:22.445 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6700 IOs, 2391772160 bytes 2026-02-22T22:07:22.490 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6800 IOs, 2422247424 bytes 2026-02-22T22:07:22.536 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6900 IOs, 2458881024 bytes 2026-02-22T22:07:22.575 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7000 IOs, 2481533952 bytes 2026-02-22T22:07:22.605 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7100 IOs, 2482144256 bytes 2026-02-22T22:07:22.646 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7200 IOs, 2506480640 bytes 2026-02-22T22:07:22.686 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7300 IOs, 2538674176 bytes 2026-02-22T22:07:22.732 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7400 IOs, 2587062272 bytes 2026-02-22T22:07:22.771 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7500 IOs, 2606973952 bytes 2026-02-22T22:07:22.821 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7600 IOs, 2648985600 bytes 2026-02-22T22:07:22.868 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7700 IOs, 2691760128 bytes 2026-02-22T22:07:22.905 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7800 IOs, 2717840384 bytes 2026-02-22T22:07:22.961 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7900 IOs, 2796592128 bytes 2026-02-22T22:07:23.034 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8000 IOs, 2894374912 bytes 2026-02-22T22:07:23.080 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8100 IOs, 2942515200 bytes 2026-02-22T22:07:23.119 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8200 IOs, 2968190976 bytes 2026-02-22T22:07:23.174 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8300 IOs, 3025622016 bytes 2026-02-22T22:07:23.223 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8400 IOs, 3081110528 bytes 2026-02-22T22:07:23.273 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8500 IOs, 3125015552 bytes 2026-02-22T22:07:23.311 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8600 IOs, 3147212800 bytes 2026-02-22T22:07:23.368 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8700 IOs, 3207997440 bytes 2026-02-22T22:07:23.434 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8800 IOs, 3293207552 bytes 2026-02-22T22:07:23.475 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8900 IOs, 3315664896 bytes 2026-02-22T22:07:23.519 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9000 IOs, 3351709696 bytes 2026-02-22T22:07:23.582 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9100 IOs, 3435204608 bytes 2026-02-22T22:07:23.612 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9200 IOs, 3442169856 bytes 2026-02-22T22:07:23.661 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9300 IOs, 3495599104 bytes 2026-02-22T22:07:23.712 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9400 IOs, 3548452864 bytes 2026-02-22T22:07:23.773 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9500 IOs, 3629614080 bytes 2026-02-22T22:07:23.826 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9600 IOs, 3691179008 bytes 2026-02-22T22:07:23.878 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9700 IOs, 3745143808 bytes 2026-02-22T22:07:23.931 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9800 IOs, 3804235776 bytes 2026-02-22T22:07:23.975 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9900 IOs, 3840932864 bytes 2026-02-22T22:07:24.013 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10000 IOs, 3859593216 bytes 2026-02-22T22:07:24.064 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10100 IOs, 3919600640 bytes 2026-02-22T22:07:24.105 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10200 IOs, 3955068928 bytes 2026-02-22T22:07:24.161 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10300 IOs, 4011744256 bytes 2026-02-22T22:07:24.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10400 IOs, 4012351488 bytes 2026-02-22T22:07:24.231 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10500 IOs, 4055066624 bytes 2026-02-22T22:07:24.266 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10600 IOs, 4068689920 bytes 2026-02-22T22:07:24.308 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10700 IOs, 4120122368 bytes 2026-02-22T22:07:24.379 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10800 IOs, 4211961856 bytes 2026-02-22T22:07:24.418 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10900 IOs, 4219912192 bytes 2026-02-22T22:07:24.457 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11000 IOs, 4241413120 bytes 2026-02-22T22:07:24.494 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11100 IOs, 4277799936 bytes 2026-02-22T22:07:24.545 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11200 IOs, 4338495488 bytes 2026-02-22T22:07:24.596 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11300 IOs, 4393550848 bytes 2026-02-22T22:07:24.639 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11400 IOs, 4428738560 bytes 2026-02-22T22:07:24.700 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11500 IOs, 4522591232 bytes 2026-02-22T22:07:24.759 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11600 IOs, 4579434496 bytes 2026-02-22T22:07:24.804 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11700 IOs, 4609961984 bytes 2026-02-22T22:07:24.840 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11800 IOs, 4625848320 bytes 2026-02-22T22:07:24.898 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11900 IOs, 4694626304 bytes 2026-02-22T22:07:24.939 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12000 IOs, 4710756352 bytes 2026-02-22T22:07:24.984 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12100 IOs, 4761163776 bytes 2026-02-22T22:07:25.042 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12200 IOs, 4841703424 bytes 2026-02-22T22:07:25.084 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12300 IOs, 4860324864 bytes 2026-02-22T22:07:25.138 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12400 IOs, 4935116800 bytes 2026-02-22T22:07:25.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12500 IOs, 4981889024 bytes 2026-02-22T22:07:25.249 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12600 IOs, 5062588416 bytes 2026-02-22T22:07:25.287 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12700 IOs, 5082077184 bytes 2026-02-22T22:07:25.334 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12800 IOs, 5130867712 bytes 2026-02-22T22:07:25.370 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12900 IOs, 5141730304 bytes 2026-02-22T22:07:25.406 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13000 IOs, 5158068224 bytes 2026-02-22T22:07:25.452 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13100 IOs, 5201168384 bytes 2026-02-22T22:07:25.490 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13200 IOs, 5224110080 bytes 2026-02-22T22:07:25.525 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13300 IOs, 5232467968 bytes 2026-02-22T22:07:25.572 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13400 IOs, 5273755648 bytes 2026-02-22T22:07:25.624 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13500 IOs, 5340645376 bytes 2026-02-22T22:07:25.680 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13600 IOs, 5406819328 bytes 2026-02-22T22:07:25.752 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13700 IOs, 5513111552 bytes 2026-02-22T22:07:25.788 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13800 IOs, 5532568576 bytes 2026-02-22T22:07:25.848 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13900 IOs, 5594819584 bytes 2026-02-22T22:07:25.884 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14000 IOs, 5611456512 bytes 2026-02-22T22:07:25.927 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14100 IOs, 5638001664 bytes 2026-02-22T22:07:25.966 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14200 IOs, 5667982336 bytes 2026-02-22T22:07:26.006 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14300 IOs, 5696029696 bytes 2026-02-22T22:07:26.045 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14400 IOs, 5712822272 bytes 2026-02-22T22:07:26.089 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14500 IOs, 5768975360 bytes 2026-02-22T22:07:26.127 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14600 IOs, 5772776448 bytes 2026-02-22T22:07:26.160 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14700 IOs, 5782374400 bytes 2026-02-22T22:07:26.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14800 IOs, 5785413632 bytes 2026-02-22T22:07:26.248 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14900 IOs, 5851372544 bytes 2026-02-22T22:07:26.285 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15000 IOs, 5868674048 bytes 2026-02-22T22:07:26.335 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15100 IOs, 5930880000 bytes 2026-02-22T22:07:26.399 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15200 IOs, 6008951808 bytes 2026-02-22T22:07:26.446 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15300 IOs, 6045239296 bytes 2026-02-22T22:07:26.492 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15400 IOs, 6079699968 bytes 2026-02-22T22:07:26.532 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15500 IOs, 6104664064 bytes 2026-02-22T22:07:26.568 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15600 IOs, 6128547840 bytes 2026-02-22T22:07:26.621 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15700 IOs, 6198989824 bytes 2026-02-22T22:07:26.681 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15800 IOs, 6258891776 bytes 2026-02-22T22:07:26.748 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15900 IOs, 6341848064 bytes 2026-02-22T22:07:26.791 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16000 IOs, 6376260608 bytes 2026-02-22T22:07:26.859 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16100 IOs, 6464567296 bytes 2026-02-22T22:07:26.897 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16200 IOs, 6505095168 bytes 2026-02-22T22:07:26.942 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16300 IOs, 6528356352 bytes 2026-02-22T22:07:26.978 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16400 IOs, 6539146240 bytes 2026-02-22T22:07:27.017 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16500 IOs, 6567694336 bytes 2026-02-22T22:07:27.061 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16600 IOs, 6598598656 bytes 2026-02-22T22:07:27.102 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16700 IOs, 6630277120 bytes 2026-02-22T22:07:27.145 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16800 IOs, 6667151360 bytes 2026-02-22T22:07:27.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16900 IOs, 6732585984 bytes 2026-02-22T22:07:27.264 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17000 IOs, 6806727680 bytes 2026-02-22T22:07:27.295 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17100 IOs, 6831265792 bytes 2026-02-22T22:07:27.348 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17200 IOs, 6879822848 bytes 2026-02-22T22:07:27.402 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17300 IOs, 6941468672 bytes 2026-02-22T22:07:27.440 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17400 IOs, 6962922496 bytes 2026-02-22T22:07:27.497 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17500 IOs, 7040327680 bytes 2026-02-22T22:07:27.550 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17600 IOs, 7094202368 bytes 2026-02-22T22:07:27.588 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17700 IOs, 7114850304 bytes 2026-02-22T22:07:27.621 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17800 IOs, 7133732864 bytes 2026-02-22T22:07:27.651 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17900 IOs, 7134328832 bytes 2026-02-22T22:07:27.681 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18000 IOs, 7138410496 bytes 2026-02-22T22:07:27.720 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18100 IOs, 7158791168 bytes 2026-02-22T22:07:27.756 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18200 IOs, 7189347328 bytes 2026-02-22T22:07:27.810 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18300 IOs, 7246567424 bytes 2026-02-22T22:07:27.862 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18400 IOs, 7286723584 bytes 2026-02-22T22:07:27.922 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18500 IOs, 7356259328 bytes 2026-02-22T22:07:27.962 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18600 IOs, 7392533504 bytes 2026-02-22T22:07:28.008 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18700 IOs, 7419842560 bytes 2026-02-22T22:07:28.038 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18800 IOs, 7427456000 bytes 2026-02-22T22:07:28.080 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18900 IOs, 7454914560 bytes 2026-02-22T22:07:28.129 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19000 IOs, 7502539776 bytes 2026-02-22T22:07:28.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19100 IOs, 7562937344 bytes 2026-02-22T22:07:28.228 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19200 IOs, 7602415616 bytes 2026-02-22T22:07:28.289 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19300 IOs, 7684983808 bytes 2026-02-22T22:07:28.340 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19400 IOs, 7733889024 bytes 2026-02-22T22:07:28.383 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19500 IOs, 7765079040 bytes 2026-02-22T22:07:28.421 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19600 IOs, 7789374464 bytes 2026-02-22T22:07:28.468 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19700 IOs, 7846388736 bytes 2026-02-22T22:07:28.542 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19800 IOs, 7933449216 bytes 2026-02-22T22:07:28.583 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19900 IOs, 7968096256 bytes 2026-02-22T22:07:28.620 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20000 IOs, 7975450624 bytes 2026-02-22T22:07:28.669 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20100 IOs, 8013519872 bytes 2026-02-22T22:07:28.732 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20200 IOs, 8087894016 bytes 2026-02-22T22:07:28.768 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20300 IOs, 8102699008 bytes 2026-02-22T22:07:28.821 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20400 IOs, 8154498048 bytes 2026-02-22T22:07:28.861 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20500 IOs, 8174662656 bytes 2026-02-22T22:07:28.913 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20600 IOs, 8258008064 bytes 2026-02-22T22:07:28.967 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20700 IOs, 8296121344 bytes 2026-02-22T22:07:29.025 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20800 IOs, 8340085760 bytes 2026-02-22T22:07:29.067 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20900 IOs, 8380130304 bytes 2026-02-22T22:07:29.120 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21000 IOs, 8431198208 bytes 2026-02-22T22:07:29.169 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21100 IOs, 8477195264 bytes 2026-02-22T22:07:29.209 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21200 IOs, 8502595584 bytes 2026-02-22T22:07:29.266 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21300 IOs, 8586776576 bytes 2026-02-22T22:07:29.322 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21400 IOs, 8628307968 bytes 2026-02-22T22:07:29.376 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21500 IOs, 8704838656 bytes 2026-02-22T22:07:29.431 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21600 IOs, 8748072960 bytes 2026-02-22T22:07:29.477 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21700 IOs, 8767106048 bytes 2026-02-22T22:07:29.535 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21800 IOs, 8846957568 bytes 2026-02-22T22:07:29.598 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21900 IOs, 8921027584 bytes 2026-02-22T22:07:29.633 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22000 IOs, 8933944320 bytes 2026-02-22T22:07:29.696 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22100 IOs, 9002845184 bytes 2026-02-22T22:07:29.732 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22200 IOs, 9021434880 bytes 2026-02-22T22:07:29.793 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22300 IOs, 9106993152 bytes 2026-02-22T22:07:29.833 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22400 IOs, 9124924416 bytes 2026-02-22T22:07:29.871 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22500 IOs, 9149907968 bytes 2026-02-22T22:07:29.903 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22600 IOs, 9151951872 bytes 2026-02-22T22:07:29.967 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22700 IOs, 9215710208 bytes 2026-02-22T22:07:30.015 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22800 IOs, 9266772992 bytes 2026-02-22T22:07:30.060 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22900 IOs, 9307424768 bytes 2026-02-22T22:07:30.103 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23000 IOs, 9335218176 bytes 2026-02-22T22:07:30.152 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23100 IOs, 9384001536 bytes 2026-02-22T22:07:30.213 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23200 IOs, 9450390528 bytes 2026-02-22T22:07:30.259 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23300 IOs, 9488413696 bytes 2026-02-22T22:07:30.307 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23400 IOs, 9536464896 bytes 2026-02-22T22:07:30.346 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23500 IOs, 9556283392 bytes 2026-02-22T22:07:30.379 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23600 IOs, 9563396096 bytes 2026-02-22T22:07:30.436 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23700 IOs, 9628160000 bytes 2026-02-22T22:07:30.486 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23800 IOs, 9669992448 bytes 2026-02-22T22:07:30.556 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23900 IOs, 9780383744 bytes 2026-02-22T22:07:30.604 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24000 IOs, 9803260928 bytes 2026-02-22T22:07:30.646 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24100 IOs, 9836119040 bytes 2026-02-22T22:07:30.692 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24200 IOs, 9873393664 bytes 2026-02-22T22:07:30.721 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24300 IOs, 9879230464 bytes 2026-02-22T22:07:30.763 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24400 IOs, 9908642816 bytes 2026-02-22T22:07:30.795 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24500 IOs, 9913446400 bytes 2026-02-22T22:07:30.833 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24600 IOs, 9950740480 bytes 2026-02-22T22:07:30.893 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24700 IOs, 9997894656 bytes 2026-02-22T22:07:30.942 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24800 IOs, 10033650688 bytes 2026-02-22T22:07:30.998 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24900 IOs, 10096571392 bytes 2026-02-22T22:07:31.052 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25000 IOs, 10149972992 bytes 2026-02-22T22:07:31.102 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25100 IOs, 10196422656 bytes 2026-02-22T22:07:31.136 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25200 IOs, 10211104768 bytes 2026-02-22T22:07:31.173 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25300 IOs, 10240401408 bytes 2026-02-22T22:07:31.228 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25400 IOs, 10283633664 bytes 2026-02-22T22:07:31.275 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25500 IOs, 10328346624 bytes 2026-02-22T22:07:31.342 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25600 IOs, 10413145088 bytes 2026-02-22T22:07:31.406 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25700 IOs, 10490792960 bytes 2026-02-22T22:07:31.439 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25800 IOs, 10499119104 bytes 2026-02-22T22:07:31.495 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25900 IOs, 10547693568 bytes 2026-02-22T22:07:31.536 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:26000 IOs, 10572175360 bytes 2026-02-22T22:07:31.570 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:26100 IOs, 10582401024 bytes 2026-02-22T22:07:31.601 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:26200 IOs, 10587770880 bytes 2026-02-22T22:07:31.641 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:26300 IOs, 10613185536 bytes 2026-02-22T22:07:31.699 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:26400 IOs, 10680187904 bytes 2026-02-22T22:07:31.744 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:26500 IOs, 10719172608 bytes 2026-02-22T22:07:31.787 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:26600 IOs, 10753647616 bytes 2026-02-22T22:07:31.838 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:26700 IOs, 10804619264 bytes 2026-02-22T22:07:31.880 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:26800 IOs, 10827467776 bytes 2026-02-22T22:07:31.917 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:26900 IOs, 10854354944 bytes 2026-02-22T22:07:31.971 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:27000 IOs, 10899949568 bytes 2026-02-22T22:07:32.025 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:27100 IOs, 10959380480 bytes 2026-02-22T22:07:32.058 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:27200 IOs, 10960497664 bytes 2026-02-22T22:07:32.087 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:27300 IOs, 10961119232 bytes 2026-02-22T22:07:32.133 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:27400 IOs, 11005008896 bytes 2026-02-22T22:07:32.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:27500 IOs, 11100378112 bytes 2026-02-22T22:07:32.263 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:27600 IOs, 11176492032 bytes 2026-02-22T22:07:32.295 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:27700 IOs, 11179991040 bytes 2026-02-22T22:07:32.338 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:27800 IOs, 11209757696 bytes 2026-02-22T22:07:32.380 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:27900 IOs, 11246090240 bytes 2026-02-22T22:07:32.421 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:28000 IOs, 11273517056 bytes 2026-02-22T22:07:32.488 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:28100 IOs, 11340637184 bytes 2026-02-22T22:07:32.519 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:28200 IOs, 11341251584 bytes 2026-02-22T22:07:32.569 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:28300 IOs, 11397536768 bytes 2026-02-22T22:07:32.608 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:28400 IOs, 11414210560 bytes 2026-02-22T22:07:32.651 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:28500 IOs, 11450496000 bytes 2026-02-22T22:07:32.690 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:28600 IOs, 11466320896 bytes 2026-02-22T22:07:32.721 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:28700 IOs, 11474551808 bytes 2026-02-22T22:07:32.773 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:28800 IOs, 11532566528 bytes 2026-02-22T22:07:32.810 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:28900 IOs, 11543215104 bytes 2026-02-22T22:07:32.869 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:29000 IOs, 11609203712 bytes 2026-02-22T22:07:32.918 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:29100 IOs, 11657036800 bytes 2026-02-22T22:07:32.978 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:29200 IOs, 11723971584 bytes 2026-02-22T22:07:33.022 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:29300 IOs, 11756398592 bytes 2026-02-22T22:07:33.056 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:29400 IOs, 11770397696 bytes 2026-02-22T22:07:33.115 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:29500 IOs, 11856277504 bytes 2026-02-22T22:07:33.175 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:29600 IOs, 11922109440 bytes 2026-02-22T22:07:33.225 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:29700 IOs, 11961341952 bytes 2026-02-22T22:07:33.283 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ error_code=124 2026-02-22T22:07:33.283 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set -e 2026-02-22T22:07:33.283 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' 124 -eq 124 ']' 2026-02-22T22:07:33.283 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:07:33.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_present cluster1 mirror image_4 present 2026-02-22T22:07:33.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:07:33.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:07:33.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_4 2026-02-22T22:07:33.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=present 2026-02-22T22:07:33.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:07:33.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:07:33.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:07:33.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_4 2026-02-22T22:07:33.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:07:33.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:07:33.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_4 2026-02-22T22:07:33.285 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_4 2026-02-22T22:07:33.285 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:07:33.310 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image_id=1260c658cef5 2026-02-22T22:07:33.310 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:07:33.310 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.1 2026-02-22T22:07:33.412 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_present cluster1 mirror image_4 present 1260c658cef5 2026-02-22T22:07:33.412 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:07:33.412 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:07:33.413 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_4 2026-02-22T22:07:33.413 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=present 2026-02-22T22:07:33.413 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id=1260c658cef5 2026-02-22T22:07:33.413 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=deleted 2026-02-22T22:07:33.413 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_image_id 2026-02-22T22:07:33.413 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_4 2026-02-22T22:07:33.413 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:07:33.413 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:07:33.413 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_4 2026-02-22T22:07:33.413 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_4 2026-02-22T22:07:33.413 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:07:33.440 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_image_id=1260c658cef5 2026-02-22T22:07:33.440 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n 1260c658cef5 2026-02-22T22:07:33.441 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -z 1260c658cef5 -o 1260c658cef5 = 1260c658cef5 2026-02-22T22:07:33.441 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_state=present 2026-02-22T22:07:33.441 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test present = present 2026-02-22T22:07:33.441 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:07:33.441 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ disable_mirror cluster2 mirror image_4 2026-02-22T22:07:33.441 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:07:33.441 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:07:33.441 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_4 2026-02-22T22:07:33.441 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_4 2026-02-22T22:07:34.285 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial026.stderr:2026-02-22T22:07:34.283+0000 7f1baf63e640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x5647246b0800 scan_remote_mirror_snapshots: failed to locate remote start snapshot: snap_id=188 2026-02-22T22:07:34.285 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial026.stderr:2026-02-22T22:07:34.283+0000 7f1baf63e640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x5647246b0800 scan_remote_mirror_snapshots: split-brain detected: failed to find matching non-primary snapshot in remote image: local_snap_id_start=188, local_snap_ns=[mirror state=non-primary, complete=1, mirror_peer_uuids=, primary_mirror_uuid=cfa81703-817a-4074-adbc-5e6e200790bf, primary_snap_id=bc, last_copied_object_number=128, snap_seqs={188=18446744073709551614}] 2026-02-22T22:07:39.288 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Mirroring disabled 2026-02-22T22:07:39.293 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_present cluster1 mirror image_4 deleted 2026-02-22T22:07:39.293 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:07:39.293 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:07:39.293 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_4 2026-02-22T22:07:39.293 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=deleted 2026-02-22T22:07:39.293 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:07:39.293 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:07:39.293 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:07:39.294 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_4 2026-02-22T22:07:39.294 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:07:39.294 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:07:39.294 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_4 2026-02-22T22:07:39.294 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_4 2026-02-22T22:07:39.294 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:07:39.319 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_4: (2) No such file or directory 2026-02-22T22:07:39.321 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image_id= 2026-02-22T22:07:39.321 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:07:39.321 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.1 2026-02-22T22:07:39.423 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_present cluster1 mirror image_4 deleted '' 2026-02-22T22:07:39.423 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:07:39.423 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:07:39.423 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_4 2026-02-22T22:07:39.423 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=deleted 2026-02-22T22:07:39.423 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:07:39.424 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=deleted 2026-02-22T22:07:39.424 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_image_id 2026-02-22T22:07:39.424 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_4 2026-02-22T22:07:39.424 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:07:39.424 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:07:39.424 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_4 2026-02-22T22:07:39.424 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_4 2026-02-22T22:07:39.424 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:07:39.445 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_4: (2) No such file or directory 2026-02-22T22:07:39.448 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_image_id= 2026-02-22T22:07:39.448 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:07:39.448 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test deleted = deleted 2026-02-22T22:07:39.448 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:07:39.448 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ purge_snapshots cluster2 mirror image_4 2026-02-22T22:07:39.448 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:07:39.449 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:07:39.449 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_4 2026-02-22T22:07:39.449 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap purge mirror/image_4 2026-02-22T22:07:39.478 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image_retry cluster2 mirror image_4 2026-02-22T22:07:39.478 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:07:39.478 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:07:39.478 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_4 2026-02-22T22:07:39.478 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:07:39.478 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0 2026-02-22T22:07:39.479 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image cluster2 mirror image_4 2026-02-22T22:07:39.479 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:07:39.479 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:07:39.480 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_4 2026-02-22T22:07:39.480 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_4 2026-02-22T22:07:39.506 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 rm mirror/image_4 2026-02-22T22:08:03.053 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2026-02-22T22:08:03.057 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:08:03.057 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 10` 2026-02-22T22:08:03.057 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_5 2026-02-22T22:08:03.057 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image cluster2 mirror image_5 512M 2026-02-22T22:08:03.057 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:08:03.057 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T22:08:03.057 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:08:03.057 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T22:08:03.057 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_5 2026-02-22T22:08:03.058 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T22:08:03.058 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local size=128 2026-02-22T22:08:03.058 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n 512M ']' 2026-02-22T22:08:03.058 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ size=512M 2026-02-22T22:08:03.058 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T22:08:03.058 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering,exclusive-lock mirror/image_5 2026-02-22T22:08:03.089 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ enable_mirror cluster2 mirror image_5 2026-02-22T22:08:03.089 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:08:03.090 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:08:03.090 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_5 2026-02-22T22:08:03.090 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T22:08:03.090 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_5 snapshot 2026-02-22T22:08:03.963 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Mirroring enabled 2026-02-22T22:08:03.968 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 info mirror/image_5 2026-02-22T22:08:03.991 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_5': 2026-02-22T22:08:03.992 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 512 MiB in 128 objects 2026-02-22T22:08:03.992 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T22:08:03.992 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 1 2026-02-22T22:08:03.992 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 1939110f917c 2026-02-22T22:08:03.992 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.1939110f917c 2026-02-22T22:08:03.992 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T22:08:03.992 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock 2026-02-22T22:08:03.992 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T22:08:03.992 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T22:08:03.993 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 22:08:03 2026 2026-02-22T22:08:03.993 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 22:08:03 2026 2026-02-22T22:08:03.993 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 22:08:03 2026 2026-02-22T22:08:03.993 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T22:08:03.993 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T22:08:03.993 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: 0dab3606-8a15-43e3-9052-ccceebca7f2b 2026-02-22T22:08:03.993 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: true 2026-02-22T22:08:03.994 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ stress_write_image cluster2 mirror image_5 2026-02-22T22:08:03.994 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:08:03.994 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:08:03.994 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_5 2026-02-22T22:08:03.995 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-22T22:08:03.995 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local duration=11 2026-02-22T22:08:03.995 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set +e 2026-02-22T22:08:03.996 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ timeout 11s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_5 --debug-rbd=20 --debug-journaler=20 2026-02-22T22:08:04.081 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:100 IOs, 59931648 bytes 2026-02-22T22:08:04.147 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:200 IOs, 169247744 bytes 2026-02-22T22:08:04.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:300 IOs, 179013632 bytes 2026-02-22T22:08:04.213 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:400 IOs, 185773056 bytes 2026-02-22T22:08:04.263 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:500 IOs, 243706880 bytes 2026-02-22T22:08:04.319 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:600 IOs, 314295296 bytes 2026-02-22T22:08:04.368 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:700 IOs, 359882752 bytes 2026-02-22T22:08:04.404 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:800 IOs, 376820736 bytes 2026-02-22T22:08:04.460 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:900 IOs, 448531456 bytes 2026-02-22T22:08:04.498 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1000 IOs, 468055040 bytes 2026-02-22T22:08:04.553 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1100 IOs, 516836352 bytes 2026-02-22T22:08:04.597 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1200 IOs, 545405952 bytes 2026-02-22T22:08:04.653 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1300 IOs, 611737600 bytes 2026-02-22T22:08:04.699 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1400 IOs, 654175232 bytes 2026-02-22T22:08:04.746 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1500 IOs, 694363136 bytes 2026-02-22T22:08:04.795 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1600 IOs, 757153792 bytes 2026-02-22T22:08:04.863 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1700 IOs, 840476672 bytes 2026-02-22T22:08:04.922 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1800 IOs, 915628032 bytes 2026-02-22T22:08:04.999 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1900 IOs, 1025002496 bytes 2026-02-22T22:08:05.064 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2000 IOs, 1100294144 bytes 2026-02-22T22:08:05.101 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2100 IOs, 1122304000 bytes 2026-02-22T22:08:05.158 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2200 IOs, 1191057408 bytes 2026-02-22T22:08:05.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2300 IOs, 1239083008 bytes 2026-02-22T22:08:05.248 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2400 IOs, 1262066688 bytes 2026-02-22T22:08:05.305 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2500 IOs, 1338700800 bytes 2026-02-22T22:08:05.351 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2600 IOs, 1398709248 bytes 2026-02-22T22:08:05.410 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2700 IOs, 1456660480 bytes 2026-02-22T22:08:05.446 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2800 IOs, 1469625344 bytes 2026-02-22T22:08:05.495 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2900 IOs, 1519172608 bytes 2026-02-22T22:08:05.538 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3000 IOs, 1554209792 bytes 2026-02-22T22:08:05.591 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3100 IOs, 1598305280 bytes 2026-02-22T22:08:05.630 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3200 IOs, 1623316480 bytes 2026-02-22T22:08:05.680 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3300 IOs, 1694092288 bytes 2026-02-22T22:08:05.716 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3400 IOs, 1696350208 bytes 2026-02-22T22:08:05.762 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3500 IOs, 1734287360 bytes 2026-02-22T22:08:05.794 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3600 IOs, 1737539584 bytes 2026-02-22T22:08:05.845 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3700 IOs, 1805548544 bytes 2026-02-22T22:08:05.891 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3800 IOs, 1831499776 bytes 2026-02-22T22:08:05.929 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3900 IOs, 1855199232 bytes 2026-02-22T22:08:05.994 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4000 IOs, 1931750400 bytes 2026-02-22T22:08:06.051 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4100 IOs, 1995086848 bytes 2026-02-22T22:08:06.116 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4200 IOs, 2090653696 bytes 2026-02-22T22:08:06.169 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4300 IOs, 2139207680 bytes 2026-02-22T22:08:06.215 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4400 IOs, 2192275456 bytes 2026-02-22T22:08:06.251 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4500 IOs, 2199498752 bytes 2026-02-22T22:08:06.314 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4600 IOs, 2293547008 bytes 2026-02-22T22:08:06.373 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4700 IOs, 2361956352 bytes 2026-02-22T22:08:06.422 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4800 IOs, 2405565440 bytes 2026-02-22T22:08:06.456 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4900 IOs, 2419972096 bytes 2026-02-22T22:08:06.487 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5000 IOs, 2434023424 bytes 2026-02-22T22:08:06.551 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5100 IOs, 2511332352 bytes 2026-02-22T22:08:06.612 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5200 IOs, 2584062976 bytes 2026-02-22T22:08:06.656 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5300 IOs, 2614272000 bytes 2026-02-22T22:08:06.699 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5400 IOs, 2653665280 bytes 2026-02-22T22:08:06.735 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5500 IOs, 2664510464 bytes 2026-02-22T22:08:06.781 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5600 IOs, 2701977600 bytes 2026-02-22T22:08:06.840 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5700 IOs, 2768795648 bytes 2026-02-22T22:08:06.882 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5800 IOs, 2798948352 bytes 2026-02-22T22:08:06.940 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5900 IOs, 2870626304 bytes 2026-02-22T22:08:06.979 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6000 IOs, 2887735296 bytes 2026-02-22T22:08:07.229 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6100 IOs, 2966749184 bytes 2026-02-22T22:08:07.271 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6200 IOs, 2996308992 bytes 2026-02-22T22:08:07.312 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6300 IOs, 3018458112 bytes 2026-02-22T22:08:07.378 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6400 IOs, 3118166016 bytes 2026-02-22T22:08:07.421 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6500 IOs, 3152717824 bytes 2026-02-22T22:08:07.464 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6600 IOs, 3187872768 bytes 2026-02-22T22:08:07.509 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6700 IOs, 3234538496 bytes 2026-02-22T22:08:07.558 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6800 IOs, 3282378752 bytes 2026-02-22T22:08:07.594 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6900 IOs, 3286544384 bytes 2026-02-22T22:08:07.638 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7000 IOs, 3323864064 bytes 2026-02-22T22:08:07.678 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7100 IOs, 3360641024 bytes 2026-02-22T22:08:07.732 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7200 IOs, 3409885184 bytes 2026-02-22T22:08:07.790 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7300 IOs, 3477447680 bytes 2026-02-22T22:08:07.833 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7400 IOs, 3507191808 bytes 2026-02-22T22:08:07.881 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7500 IOs, 3547046912 bytes 2026-02-22T22:08:07.920 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7600 IOs, 3574893568 bytes 2026-02-22T22:08:07.953 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7700 IOs, 3577949184 bytes 2026-02-22T22:08:07.988 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7800 IOs, 3584381952 bytes 2026-02-22T22:08:08.041 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7900 IOs, 3639560192 bytes 2026-02-22T22:08:08.092 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8000 IOs, 3710919680 bytes 2026-02-22T22:08:08.157 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8100 IOs, 3770042368 bytes 2026-02-22T22:08:08.190 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8200 IOs, 3779163136 bytes 2026-02-22T22:08:08.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8300 IOs, 3806753792 bytes 2026-02-22T22:08:08.303 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8400 IOs, 3918926848 bytes 2026-02-22T22:08:08.345 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8500 IOs, 3967700992 bytes 2026-02-22T22:08:08.387 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8600 IOs, 3983040512 bytes 2026-02-22T22:08:08.444 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8700 IOs, 4047660032 bytes 2026-02-22T22:08:08.484 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8800 IOs, 4071989248 bytes 2026-02-22T22:08:08.561 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8900 IOs, 4181079040 bytes 2026-02-22T22:08:08.616 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9000 IOs, 4243562496 bytes 2026-02-22T22:08:08.658 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9100 IOs, 4277690368 bytes 2026-02-22T22:08:08.709 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9200 IOs, 4329399296 bytes 2026-02-22T22:08:08.767 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9300 IOs, 4416567296 bytes 2026-02-22T22:08:08.826 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9400 IOs, 4483596288 bytes 2026-02-22T22:08:08.875 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9500 IOs, 4523108352 bytes 2026-02-22T22:08:08.919 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9600 IOs, 4560768000 bytes 2026-02-22T22:08:08.991 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9700 IOs, 4670329856 bytes 2026-02-22T22:08:09.064 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9800 IOs, 4762273792 bytes 2026-02-22T22:08:09.114 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9900 IOs, 4812729344 bytes 2026-02-22T22:08:09.155 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10000 IOs, 4831847424 bytes 2026-02-22T22:08:09.196 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10100 IOs, 4860696576 bytes 2026-02-22T22:08:09.255 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10200 IOs, 4939626496 bytes 2026-02-22T22:08:09.303 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10300 IOs, 4990768128 bytes 2026-02-22T22:08:09.346 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10400 IOs, 5026672640 bytes 2026-02-22T22:08:09.389 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10500 IOs, 5056846848 bytes 2026-02-22T22:08:09.436 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10600 IOs, 5095088128 bytes 2026-02-22T22:08:09.481 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10700 IOs, 5138508800 bytes 2026-02-22T22:08:09.518 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10800 IOs, 5156200448 bytes 2026-02-22T22:08:09.562 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10900 IOs, 5196971008 bytes 2026-02-22T22:08:09.616 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11000 IOs, 5243464704 bytes 2026-02-22T22:08:09.674 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11100 IOs, 5317424128 bytes 2026-02-22T22:08:09.709 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11200 IOs, 5323574272 bytes 2026-02-22T22:08:09.759 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11300 IOs, 5382171648 bytes 2026-02-22T22:08:09.815 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11400 IOs, 5449788416 bytes 2026-02-22T22:08:09.865 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11500 IOs, 5503384576 bytes 2026-02-22T22:08:09.901 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11600 IOs, 5521121280 bytes 2026-02-22T22:08:09.984 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11700 IOs, 5639138304 bytes 2026-02-22T22:08:10.035 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11800 IOs, 5691776000 bytes 2026-02-22T22:08:10.082 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11900 IOs, 5718976512 bytes 2026-02-22T22:08:10.139 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12000 IOs, 5785767936 bytes 2026-02-22T22:08:10.175 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12100 IOs, 5816751104 bytes 2026-02-22T22:08:10.224 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12200 IOs, 5848928256 bytes 2026-02-22T22:08:10.274 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12300 IOs, 5885892608 bytes 2026-02-22T22:08:10.306 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12400 IOs, 5886521344 bytes 2026-02-22T22:08:10.340 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12500 IOs, 5896795136 bytes 2026-02-22T22:08:10.392 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12600 IOs, 5965128704 bytes 2026-02-22T22:08:10.442 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12700 IOs, 5997891584 bytes 2026-02-22T22:08:10.487 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12800 IOs, 6036112384 bytes 2026-02-22T22:08:10.533 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12900 IOs, 6080236544 bytes 2026-02-22T22:08:10.577 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13000 IOs, 6108138496 bytes 2026-02-22T22:08:10.617 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13100 IOs, 6126757888 bytes 2026-02-22T22:08:10.681 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13200 IOs, 6216700928 bytes 2026-02-22T22:08:10.738 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13300 IOs, 6278499328 bytes 2026-02-22T22:08:10.768 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13400 IOs, 6279112704 bytes 2026-02-22T22:08:10.806 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13500 IOs, 6303101952 bytes 2026-02-22T22:08:11.058 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13600 IOs, 6390813696 bytes 2026-02-22T22:08:11.109 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13700 IOs, 6450275328 bytes 2026-02-22T22:08:11.160 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13800 IOs, 6492290048 bytes 2026-02-22T22:08:11.214 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13900 IOs, 6562822144 bytes 2026-02-22T22:08:11.264 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14000 IOs, 6597865472 bytes 2026-02-22T22:08:11.327 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14100 IOs, 6696353792 bytes 2026-02-22T22:08:11.383 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14200 IOs, 6745958400 bytes 2026-02-22T22:08:11.444 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14300 IOs, 6819768320 bytes 2026-02-22T22:08:11.492 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14400 IOs, 6874104832 bytes 2026-02-22T22:08:11.523 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14500 IOs, 6881021952 bytes 2026-02-22T22:08:11.576 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14600 IOs, 6945262592 bytes 2026-02-22T22:08:11.625 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14700 IOs, 6990359552 bytes 2026-02-22T22:08:11.683 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14800 IOs, 7075950592 bytes 2026-02-22T22:08:11.729 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14900 IOs, 7089843200 bytes 2026-02-22T22:08:11.806 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15000 IOs, 7203423232 bytes 2026-02-22T22:08:11.849 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15100 IOs, 7238543360 bytes 2026-02-22T22:08:11.883 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15200 IOs, 7250299904 bytes 2026-02-22T22:08:11.919 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15300 IOs, 7266364416 bytes 2026-02-22T22:08:11.979 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15400 IOs, 7352094720 bytes 2026-02-22T22:08:12.039 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15500 IOs, 7402597376 bytes 2026-02-22T22:08:12.091 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15600 IOs, 7444571136 bytes 2026-02-22T22:08:12.135 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15700 IOs, 7485312000 bytes 2026-02-22T22:08:12.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15800 IOs, 7527046144 bytes 2026-02-22T22:08:12.218 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15900 IOs, 7543348224 bytes 2026-02-22T22:08:12.259 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16000 IOs, 7571959808 bytes 2026-02-22T22:08:12.297 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16100 IOs, 7589897216 bytes 2026-02-22T22:08:12.354 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16200 IOs, 7672213504 bytes 2026-02-22T22:08:12.397 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16300 IOs, 7676180480 bytes 2026-02-22T22:08:12.465 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16400 IOs, 7765137408 bytes 2026-02-22T22:08:12.505 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16500 IOs, 7791157248 bytes 2026-02-22T22:08:12.569 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16600 IOs, 7887262720 bytes 2026-02-22T22:08:12.614 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16700 IOs, 7907559424 bytes 2026-02-22T22:08:12.658 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16800 IOs, 7941568512 bytes 2026-02-22T22:08:12.698 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16900 IOs, 7966957568 bytes 2026-02-22T22:08:12.741 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17000 IOs, 8008182784 bytes 2026-02-22T22:08:12.797 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17100 IOs, 8073764864 bytes 2026-02-22T22:08:12.841 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17200 IOs, 8106646528 bytes 2026-02-22T22:08:12.887 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17300 IOs, 8156660736 bytes 2026-02-22T22:08:12.943 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17400 IOs, 8212247552 bytes 2026-02-22T22:08:12.992 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17500 IOs, 8275407872 bytes 2026-02-22T22:08:13.031 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17600 IOs, 8293426176 bytes 2026-02-22T22:08:13.070 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17700 IOs, 8313868288 bytes 2026-02-22T22:08:13.110 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17800 IOs, 8335267840 bytes 2026-02-22T22:08:13.151 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17900 IOs, 8355778560 bytes 2026-02-22T22:08:13.224 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18000 IOs, 8480832512 bytes 2026-02-22T22:08:13.261 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18100 IOs, 8489257984 bytes 2026-02-22T22:08:13.323 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18200 IOs, 8561465344 bytes 2026-02-22T22:08:13.354 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18300 IOs, 8566557696 bytes 2026-02-22T22:08:13.393 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18400 IOs, 8589224960 bytes 2026-02-22T22:08:13.439 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18500 IOs, 8645643264 bytes 2026-02-22T22:08:13.503 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18600 IOs, 8723631104 bytes 2026-02-22T22:08:13.552 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18700 IOs, 8760415232 bytes 2026-02-22T22:08:13.628 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18800 IOs, 8862996480 bytes 2026-02-22T22:08:13.664 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18900 IOs, 8871868416 bytes 2026-02-22T22:08:13.702 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19000 IOs, 8890092544 bytes 2026-02-22T22:08:13.758 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19100 IOs, 8957743104 bytes 2026-02-22T22:08:13.806 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19200 IOs, 8993488896 bytes 2026-02-22T22:08:13.860 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19300 IOs, 9057192960 bytes 2026-02-22T22:08:13.899 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19400 IOs, 9066962944 bytes 2026-02-22T22:08:13.935 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19500 IOs, 9075280896 bytes 2026-02-22T22:08:13.966 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19600 IOs, 9075909632 bytes 2026-02-22T22:08:14.026 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19700 IOs, 9136339968 bytes 2026-02-22T22:08:14.066 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19800 IOs, 9161180160 bytes 2026-02-22T22:08:14.101 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19900 IOs, 9170338816 bytes 2026-02-22T22:08:14.144 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20000 IOs, 9210189824 bytes 2026-02-22T22:08:14.194 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20100 IOs, 9256491008 bytes 2026-02-22T22:08:14.242 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20200 IOs, 9297612800 bytes 2026-02-22T22:08:14.307 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20300 IOs, 9373229056 bytes 2026-02-22T22:08:14.379 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20400 IOs, 9481442304 bytes 2026-02-22T22:08:14.410 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20500 IOs, 9482060800 bytes 2026-02-22T22:08:14.458 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20600 IOs, 9546253312 bytes 2026-02-22T22:08:14.511 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20700 IOs, 9584307200 bytes 2026-02-22T22:08:14.559 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20800 IOs, 9618967552 bytes 2026-02-22T22:08:14.590 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20900 IOs, 9619588096 bytes 2026-02-22T22:08:14.618 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21000 IOs, 9620199424 bytes 2026-02-22T22:08:14.686 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21100 IOs, 9710631936 bytes 2026-02-22T22:08:14.734 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21200 IOs, 9756489728 bytes 2026-02-22T22:08:14.793 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21300 IOs, 9846432768 bytes 2026-02-22T22:08:14.839 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21400 IOs, 9874432000 bytes 2026-02-22T22:08:14.876 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21500 IOs, 9901556736 bytes 2026-02-22T22:08:14.927 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21600 IOs, 9946558464 bytes 2026-02-22T22:08:14.964 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21700 IOs, 9965394944 bytes 2026-02-22T22:08:15.000 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ error_code=124 2026-02-22T22:08:15.001 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set -e 2026-02-22T22:08:15.001 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' 124 -eq 124 ']' 2026-02-22T22:08:15.001 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:08:15.001 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_present cluster1 mirror image_5 present 2026-02-22T22:08:15.001 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:08:15.001 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:08:15.001 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_5 2026-02-22T22:08:15.001 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=present 2026-02-22T22:08:15.001 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:08:15.001 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:08:15.002 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:08:15.002 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_5 2026-02-22T22:08:15.002 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:08:15.002 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:08:15.002 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_5 2026-02-22T22:08:15.002 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_5 2026-02-22T22:08:15.002 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:08:15.029 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image_id=1260e5a7db3d 2026-02-22T22:08:15.029 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:08:15.029 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.1 2026-02-22T22:08:15.130 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_present cluster1 mirror image_5 present 1260e5a7db3d 2026-02-22T22:08:15.130 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:08:15.131 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:08:15.131 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_5 2026-02-22T22:08:15.131 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=present 2026-02-22T22:08:15.131 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id=1260e5a7db3d 2026-02-22T22:08:15.131 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=deleted 2026-02-22T22:08:15.131 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_image_id 2026-02-22T22:08:15.131 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_5 2026-02-22T22:08:15.131 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:08:15.131 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:08:15.131 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_5 2026-02-22T22:08:15.131 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_5 2026-02-22T22:08:15.132 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:08:15.159 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_image_id=1260e5a7db3d 2026-02-22T22:08:15.159 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n 1260e5a7db3d 2026-02-22T22:08:15.159 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -z 1260e5a7db3d -o 1260e5a7db3d = 1260e5a7db3d 2026-02-22T22:08:15.159 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_state=present 2026-02-22T22:08:15.159 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test present = present 2026-02-22T22:08:15.159 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:08:15.160 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ disable_mirror cluster2 mirror image_5 2026-02-22T22:08:15.160 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:08:15.160 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:08:15.160 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_5 2026-02-22T22:08:15.160 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_5 2026-02-22T22:08:16.103 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial026.stderr:2026-02-22T22:08:16.101+0000 7f1baf63e640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x564724b22000 scan_remote_mirror_snapshots: failed to locate remote start snapshot: snap_id=189 2026-02-22T22:08:16.103 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial026.stderr:2026-02-22T22:08:16.101+0000 7f1baf63e640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x564724b22000 scan_remote_mirror_snapshots: split-brain detected: failed to find matching non-primary snapshot in remote image: local_snap_id_start=189, local_snap_ns=[mirror state=non-primary, complete=1, mirror_peer_uuids=, primary_mirror_uuid=cfa81703-817a-4074-adbc-5e6e200790bf, primary_snap_id=bd, last_copied_object_number=128, snap_seqs={189=18446744073709551614}] 2026-02-22T22:08:21.102 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Mirroring disabled 2026-02-22T22:08:21.109 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_present cluster1 mirror image_5 deleted 2026-02-22T22:08:21.110 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:08:21.110 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:08:21.110 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_5 2026-02-22T22:08:21.110 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=deleted 2026-02-22T22:08:21.110 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:08:21.110 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:08:21.110 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:08:21.110 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_5 2026-02-22T22:08:21.110 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:08:21.111 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:08:21.111 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_5 2026-02-22T22:08:21.111 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_5 2026-02-22T22:08:21.111 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:08:21.134 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_5: (2) No such file or directory 2026-02-22T22:08:21.136 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image_id= 2026-02-22T22:08:21.136 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:08:21.136 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.1 2026-02-22T22:08:21.237 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_present cluster1 mirror image_5 deleted '' 2026-02-22T22:08:21.237 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:08:21.237 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:08:21.238 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_5 2026-02-22T22:08:21.238 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=deleted 2026-02-22T22:08:21.238 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:08:21.238 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=deleted 2026-02-22T22:08:21.238 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_image_id 2026-02-22T22:08:21.238 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_5 2026-02-22T22:08:21.238 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:08:21.238 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:08:21.238 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_5 2026-02-22T22:08:21.238 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_5 2026-02-22T22:08:21.238 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:08:21.261 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_5: (2) No such file or directory 2026-02-22T22:08:21.263 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_image_id= 2026-02-22T22:08:21.263 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:08:21.263 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test deleted = deleted 2026-02-22T22:08:21.263 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:08:21.263 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ purge_snapshots cluster2 mirror image_5 2026-02-22T22:08:21.264 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:08:21.264 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:08:21.264 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_5 2026-02-22T22:08:21.264 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap purge mirror/image_5 2026-02-22T22:08:21.294 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image_retry cluster2 mirror image_5 2026-02-22T22:08:21.294 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:08:21.294 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:08:21.294 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_5 2026-02-22T22:08:21.294 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:08:21.294 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0 2026-02-22T22:08:21.294 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image cluster2 mirror image_5 2026-02-22T22:08:21.295 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:08:21.295 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:08:21.295 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_5 2026-02-22T22:08:21.295 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_5 2026-02-22T22:08:21.323 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 rm mirror/image_5 2026-02-22T22:08:25.006 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.5422382342959485) > weight (0.25) 2026-02-22T22:08:25.006 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.9511909834631531) > weight (0.25) 2026-02-22T22:08:25.006 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.48480254839922876) > weight (0.25) 2026-02-22T22:08:25.007 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.3 2026-02-22T22:08:25.007 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Sent signal 15 2026-02-22T22:08:25.007 INFO:tasks.rbd_mirror_thrash:waiting for 9.165951415416716 secs before reviving daemons 2026-02-22T22:08:25.008 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial026.stderr:2026-02-22T22:08:25.006+0000 7f44a5f58640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.3 (PID: 81008) UID: 1000 2026-02-22T22:08:34.174 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.3 2026-02-22T22:08:34.174 INFO:teuthology.orchestra.run:waiting for 600 2026-02-22T22:08:34.174 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.3 2026-02-22T22:08:34.175 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Restarting daemon 2026-02-22T22:08:34.175 DEBUG:teuthology.orchestra.run.trial026:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.3 2026-02-22T22:08:34.177 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Started 2026-02-22T22:08:34.177 INFO:tasks.rbd_mirror_thrash:waiting for 114.7203091086685 secs before thrashing 2026-02-22T22:08:44.340 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2026-02-22T22:08:44.342 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:08:44.342 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 10` 2026-02-22T22:08:44.342 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_6 2026-02-22T22:08:44.342 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image cluster2 mirror image_6 512M 2026-02-22T22:08:44.343 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:08:44.343 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T22:08:44.343 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:08:44.343 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T22:08:44.343 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_6 2026-02-22T22:08:44.343 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T22:08:44.343 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local size=128 2026-02-22T22:08:44.343 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n 512M ']' 2026-02-22T22:08:44.343 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ size=512M 2026-02-22T22:08:44.343 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T22:08:44.343 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering,exclusive-lock mirror/image_6 2026-02-22T22:08:44.374 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ enable_mirror cluster2 mirror image_6 2026-02-22T22:08:44.374 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:08:44.374 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:08:44.374 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_6 2026-02-22T22:08:44.374 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T22:08:44.374 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_6 snapshot 2026-02-22T22:08:45.247 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Mirroring enabled 2026-02-22T22:08:45.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 info mirror/image_6 2026-02-22T22:08:45.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_6': 2026-02-22T22:08:45.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 512 MiB in 128 objects 2026-02-22T22:08:45.278 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T22:08:45.279 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 1 2026-02-22T22:08:45.279 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 1955d33f306b 2026-02-22T22:08:45.279 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.1955d33f306b 2026-02-22T22:08:45.279 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T22:08:45.279 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock 2026-02-22T22:08:45.279 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T22:08:45.279 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T22:08:45.279 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 22:08:44 2026 2026-02-22T22:08:45.279 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 22:08:44 2026 2026-02-22T22:08:45.279 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 22:08:44 2026 2026-02-22T22:08:45.279 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T22:08:45.280 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T22:08:45.280 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: 2b6752ab-b7b3-4201-a3a1-6ac5ee5ce46e 2026-02-22T22:08:45.280 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: true 2026-02-22T22:08:45.280 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ stress_write_image cluster2 mirror image_6 2026-02-22T22:08:45.281 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:08:45.281 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:08:45.281 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_6 2026-02-22T22:08:45.281 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-22T22:08:45.282 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local duration=11 2026-02-22T22:08:45.282 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set +e 2026-02-22T22:08:45.282 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ timeout 11s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_6 --debug-rbd=20 --debug-journaler=20 2026-02-22T22:08:45.357 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:100 IOs, 30504960 bytes 2026-02-22T22:08:45.423 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:200 IOs, 139890688 bytes 2026-02-22T22:08:45.481 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:300 IOs, 238473216 bytes 2026-02-22T22:08:45.544 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:400 IOs, 305860608 bytes 2026-02-22T22:08:45.603 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:500 IOs, 381962240 bytes 2026-02-22T22:08:45.649 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:600 IOs, 445088768 bytes 2026-02-22T22:08:45.685 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:700 IOs, 445686784 bytes 2026-02-22T22:08:45.750 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:800 IOs, 545321984 bytes 2026-02-22T22:08:45.803 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:900 IOs, 618622976 bytes 2026-02-22T22:08:45.844 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1000 IOs, 637960192 bytes 2026-02-22T22:08:45.893 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1100 IOs, 696601600 bytes 2026-02-22T22:08:45.943 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1200 IOs, 744525824 bytes 2026-02-22T22:08:45.993 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1300 IOs, 809986048 bytes 2026-02-22T22:08:46.049 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1400 IOs, 880199680 bytes 2026-02-22T22:08:46.092 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1500 IOs, 907717632 bytes 2026-02-22T22:08:46.121 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1600 IOs, 908331008 bytes 2026-02-22T22:08:46.178 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1700 IOs, 980444160 bytes 2026-02-22T22:08:46.218 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1800 IOs, 1007239168 bytes 2026-02-22T22:08:46.259 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1900 IOs, 1026455552 bytes 2026-02-22T22:08:46.299 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2000 IOs, 1060525056 bytes 2026-02-22T22:08:46.347 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2100 IOs, 1101157376 bytes 2026-02-22T22:08:46.379 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2200 IOs, 1111593984 bytes 2026-02-22T22:08:46.425 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2300 IOs, 1156206592 bytes 2026-02-22T22:08:46.470 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2400 IOs, 1203191808 bytes 2026-02-22T22:08:46.521 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2500 IOs, 1270358016 bytes 2026-02-22T22:08:46.576 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2600 IOs, 1345615872 bytes 2026-02-22T22:08:46.648 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2700 IOs, 1443826688 bytes 2026-02-22T22:08:46.693 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2800 IOs, 1477065728 bytes 2026-02-22T22:08:46.741 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2900 IOs, 1532647424 bytes 2026-02-22T22:08:46.792 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3000 IOs, 1580871680 bytes 2026-02-22T22:08:46.855 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3100 IOs, 1666021376 bytes 2026-02-22T22:08:46.905 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3200 IOs, 1716032512 bytes 2026-02-22T22:08:46.945 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3300 IOs, 1740633088 bytes 2026-02-22T22:08:47.009 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3400 IOs, 1829460992 bytes 2026-02-22T22:08:47.053 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3500 IOs, 1862589440 bytes 2026-02-22T22:08:47.085 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3600 IOs, 1863225344 bytes 2026-02-22T22:08:47.137 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3700 IOs, 1928041472 bytes 2026-02-22T22:08:47.188 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3800 IOs, 1975279616 bytes 2026-02-22T22:08:47.420 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3900 IOs, 1992557568 bytes 2026-02-22T22:08:47.471 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4000 IOs, 2049635328 bytes 2026-02-22T22:08:47.522 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4100 IOs, 2100041728 bytes 2026-02-22T22:08:47.557 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4200 IOs, 2127008768 bytes 2026-02-22T22:08:47.618 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4300 IOs, 2202484736 bytes 2026-02-22T22:08:47.653 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4400 IOs, 2215136256 bytes 2026-02-22T22:08:47.690 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4500 IOs, 2229598208 bytes 2026-02-22T22:08:47.742 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4600 IOs, 2282695680 bytes 2026-02-22T22:08:47.811 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4700 IOs, 2368392192 bytes 2026-02-22T22:08:47.861 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4800 IOs, 2418660352 bytes 2026-02-22T22:08:47.903 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4900 IOs, 2453000192 bytes 2026-02-22T22:08:47.945 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5000 IOs, 2488652800 bytes 2026-02-22T22:08:47.989 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5100 IOs, 2529210368 bytes 2026-02-22T22:08:48.023 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5200 IOs, 2568905728 bytes 2026-02-22T22:08:48.073 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5300 IOs, 2592233472 bytes 2026-02-22T22:08:48.103 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5400 IOs, 2594836480 bytes 2026-02-22T22:08:48.140 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5500 IOs, 2609563648 bytes 2026-02-22T22:08:48.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5600 IOs, 2655597568 bytes 2026-02-22T22:08:48.235 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5700 IOs, 2697105408 bytes 2026-02-22T22:08:48.301 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5800 IOs, 2792039424 bytes 2026-02-22T22:08:48.358 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5900 IOs, 2856457216 bytes 2026-02-22T22:08:48.393 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6000 IOs, 2871322624 bytes 2026-02-22T22:08:48.439 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6100 IOs, 2916587520 bytes 2026-02-22T22:08:48.468 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6200 IOs, 2917970944 bytes 2026-02-22T22:08:48.518 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6300 IOs, 2968203264 bytes 2026-02-22T22:08:48.562 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6400 IOs, 3012099072 bytes 2026-02-22T22:08:48.625 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6500 IOs, 3083101184 bytes 2026-02-22T22:08:48.673 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6600 IOs, 3133410304 bytes 2026-02-22T22:08:48.720 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6700 IOs, 3178120192 bytes 2026-02-22T22:08:48.778 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6800 IOs, 3252118528 bytes 2026-02-22T22:08:48.813 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6900 IOs, 3268442112 bytes 2026-02-22T22:08:48.851 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7000 IOs, 3293660160 bytes 2026-02-22T22:08:48.881 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7100 IOs, 3294280704 bytes 2026-02-22T22:08:48.923 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7200 IOs, 3338419200 bytes 2026-02-22T22:08:48.977 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7300 IOs, 3403615232 bytes 2026-02-22T22:08:49.023 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7400 IOs, 3435572224 bytes 2026-02-22T22:08:49.085 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7500 IOs, 3518701568 bytes 2026-02-22T22:08:49.148 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7600 IOs, 3605860352 bytes 2026-02-22T22:08:49.189 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7700 IOs, 3626235904 bytes 2026-02-22T22:08:49.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7800 IOs, 3650845696 bytes 2026-02-22T22:08:49.269 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7900 IOs, 3700451328 bytes 2026-02-22T22:08:49.327 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8000 IOs, 3757751296 bytes 2026-02-22T22:08:49.357 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8100 IOs, 3761876992 bytes 2026-02-22T22:08:49.402 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8200 IOs, 3823490048 bytes 2026-02-22T22:08:49.455 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8300 IOs, 3874613248 bytes 2026-02-22T22:08:49.490 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8400 IOs, 3889531904 bytes 2026-02-22T22:08:49.544 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8500 IOs, 3952361472 bytes 2026-02-22T22:08:49.580 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8600 IOs, 3966566400 bytes 2026-02-22T22:08:49.634 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8700 IOs, 4033882112 bytes 2026-02-22T22:08:49.671 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8800 IOs, 4050401280 bytes 2026-02-22T22:08:49.721 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8900 IOs, 4093473792 bytes 2026-02-22T22:08:49.772 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9000 IOs, 4143459328 bytes 2026-02-22T22:08:49.816 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9100 IOs, 4178245632 bytes 2026-02-22T22:08:49.860 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9200 IOs, 4224757760 bytes 2026-02-22T22:08:49.912 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9300 IOs, 4275549184 bytes 2026-02-22T22:08:49.944 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9400 IOs, 4278721536 bytes 2026-02-22T22:08:49.997 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9500 IOs, 4338048000 bytes 2026-02-22T22:08:50.040 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9600 IOs, 4370260992 bytes 2026-02-22T22:08:50.082 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9700 IOs, 4408786944 bytes 2026-02-22T22:08:50.131 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9800 IOs, 4448955392 bytes 2026-02-22T22:08:50.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9900 IOs, 4460614656 bytes 2026-02-22T22:08:50.214 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10000 IOs, 4506145792 bytes 2026-02-22T22:08:50.259 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10100 IOs, 4561433600 bytes 2026-02-22T22:08:50.309 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10200 IOs, 4591703040 bytes 2026-02-22T22:08:50.366 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10300 IOs, 4663008256 bytes 2026-02-22T22:08:50.417 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10400 IOs, 4710672384 bytes 2026-02-22T22:08:50.462 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10500 IOs, 4756162560 bytes 2026-02-22T22:08:50.534 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10600 IOs, 4846660608 bytes 2026-02-22T22:08:50.573 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10700 IOs, 4867837952 bytes 2026-02-22T22:08:50.608 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10800 IOs, 4884637696 bytes 2026-02-22T22:08:50.649 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10900 IOs, 4910396416 bytes 2026-02-22T22:08:50.699 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11000 IOs, 4963708928 bytes 2026-02-22T22:08:50.739 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11100 IOs, 4984628224 bytes 2026-02-22T22:08:50.782 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11200 IOs, 5020008448 bytes 2026-02-22T22:08:50.846 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11300 IOs, 5109078016 bytes 2026-02-22T22:08:50.892 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11400 IOs, 5137607680 bytes 2026-02-22T22:08:50.937 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11500 IOs, 5220718592 bytes 2026-02-22T22:08:51.013 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11600 IOs, 5290510336 bytes 2026-02-22T22:08:51.051 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11700 IOs, 5324438528 bytes 2026-02-22T22:08:51.109 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11800 IOs, 5374566400 bytes 2026-02-22T22:08:51.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11900 IOs, 5504307200 bytes 2026-02-22T22:08:51.232 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12000 IOs, 5514626048 bytes 2026-02-22T22:08:51.279 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12100 IOs, 5555167232 bytes 2026-02-22T22:08:51.318 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12200 IOs, 5597979648 bytes 2026-02-22T22:08:51.374 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12300 IOs, 5637229568 bytes 2026-02-22T22:08:51.408 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12400 IOs, 5650919424 bytes 2026-02-22T22:08:51.445 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12500 IOs, 5667730432 bytes 2026-02-22T22:08:51.487 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12600 IOs, 5715426304 bytes 2026-02-22T22:08:51.534 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12700 IOs, 5752889344 bytes 2026-02-22T22:08:51.569 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12800 IOs, 5754886144 bytes 2026-02-22T22:08:51.606 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12900 IOs, 5782902784 bytes 2026-02-22T22:08:51.654 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13000 IOs, 5826318336 bytes 2026-02-22T22:08:51.707 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13100 IOs, 5873067008 bytes 2026-02-22T22:08:51.752 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13200 IOs, 5905688576 bytes 2026-02-22T22:08:51.795 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13300 IOs, 5936740352 bytes 2026-02-22T22:08:51.835 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13400 IOs, 5962811392 bytes 2026-02-22T22:08:51.884 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13500 IOs, 6009860096 bytes 2026-02-22T22:08:51.939 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13600 IOs, 6065887232 bytes 2026-02-22T22:08:51.992 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13700 IOs, 6115902464 bytes 2026-02-22T22:08:52.055 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13800 IOs, 6194330624 bytes 2026-02-22T22:08:52.097 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13900 IOs, 6219121664 bytes 2026-02-22T22:08:52.135 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14000 IOs, 6242365440 bytes 2026-02-22T22:08:52.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14100 IOs, 6267522048 bytes 2026-02-22T22:08:52.227 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14200 IOs, 6315727872 bytes 2026-02-22T22:08:52.273 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14300 IOs, 6359267328 bytes 2026-02-22T22:08:52.326 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14400 IOs, 6419718144 bytes 2026-02-22T22:08:52.363 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14500 IOs, 6438346752 bytes 2026-02-22T22:08:52.426 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14600 IOs, 6526571520 bytes 2026-02-22T22:08:52.470 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14700 IOs, 6557906944 bytes 2026-02-22T22:08:52.505 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14800 IOs, 6567511040 bytes 2026-02-22T22:08:52.538 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14900 IOs, 6585700352 bytes 2026-02-22T22:08:52.591 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15000 IOs, 6641434624 bytes 2026-02-22T22:08:52.621 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15100 IOs, 6642029568 bytes 2026-02-22T22:08:52.653 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15200 IOs, 6652767232 bytes 2026-02-22T22:08:52.698 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15300 IOs, 6692074496 bytes 2026-02-22T22:08:52.737 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15400 IOs, 6711620608 bytes 2026-02-22T22:08:52.788 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15500 IOs, 6768971776 bytes 2026-02-22T22:08:52.830 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15600 IOs, 6791262208 bytes 2026-02-22T22:08:52.876 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15700 IOs, 6832806912 bytes 2026-02-22T22:08:52.930 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15800 IOs, 6881409024 bytes 2026-02-22T22:08:52.987 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15900 IOs, 6945811456 bytes 2026-02-22T22:08:53.046 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16000 IOs, 7010176000 bytes 2026-02-22T22:08:53.115 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16100 IOs, 7089889280 bytes 2026-02-22T22:08:53.146 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16200 IOs, 7096467456 bytes 2026-02-22T22:08:53.181 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16300 IOs, 7105705984 bytes 2026-02-22T22:08:53.226 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16400 IOs, 7149017088 bytes 2026-02-22T22:08:53.262 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16500 IOs, 7161724928 bytes 2026-02-22T22:08:53.305 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16600 IOs, 7192330240 bytes 2026-02-22T22:08:53.339 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16700 IOs, 7202825216 bytes 2026-02-22T22:08:53.394 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16800 IOs, 7270949888 bytes 2026-02-22T22:08:53.471 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16900 IOs, 7376352256 bytes 2026-02-22T22:08:53.521 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17000 IOs, 7428850688 bytes 2026-02-22T22:08:53.582 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17100 IOs, 7503230976 bytes 2026-02-22T22:08:53.655 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17200 IOs, 7626106880 bytes 2026-02-22T22:08:53.706 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17300 IOs, 7656701952 bytes 2026-02-22T22:08:53.769 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17400 IOs, 7733712896 bytes 2026-02-22T22:08:53.815 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17500 IOs, 7773335552 bytes 2026-02-22T22:08:53.859 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17600 IOs, 7826910208 bytes 2026-02-22T22:08:53.901 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17700 IOs, 7829202944 bytes 2026-02-22T22:08:53.974 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17800 IOs, 7944423424 bytes 2026-02-22T22:08:54.035 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17900 IOs, 8011318272 bytes 2026-02-22T22:08:54.066 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18000 IOs, 8011902976 bytes 2026-02-22T22:08:54.120 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18100 IOs, 8071567360 bytes 2026-02-22T22:08:54.173 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18200 IOs, 8129053696 bytes 2026-02-22T22:08:54.219 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18300 IOs, 8166014976 bytes 2026-02-22T22:08:54.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18400 IOs, 8183370752 bytes 2026-02-22T22:08:54.301 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18500 IOs, 8228800512 bytes 2026-02-22T22:08:54.354 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18600 IOs, 8286062592 bytes 2026-02-22T22:08:54.401 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18700 IOs, 8332599296 bytes 2026-02-22T22:08:54.446 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18800 IOs, 8377009152 bytes 2026-02-22T22:08:54.484 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18900 IOs, 8400410624 bytes 2026-02-22T22:08:54.520 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19000 IOs, 8415806464 bytes 2026-02-22T22:08:54.559 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19100 IOs, 8435078144 bytes 2026-02-22T22:08:54.600 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19200 IOs, 8463439872 bytes 2026-02-22T22:08:54.660 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19300 IOs, 8537319424 bytes 2026-02-22T22:08:54.701 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19400 IOs, 8578655232 bytes 2026-02-22T22:08:54.759 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19500 IOs, 8622733312 bytes 2026-02-22T22:08:54.802 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19600 IOs, 8658259968 bytes 2026-02-22T22:08:54.841 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19700 IOs, 8687650816 bytes 2026-02-22T22:08:54.883 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19800 IOs, 8712885248 bytes 2026-02-22T22:08:54.924 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19900 IOs, 8737284096 bytes 2026-02-22T22:08:54.968 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20000 IOs, 8770982912 bytes 2026-02-22T22:08:55.040 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20100 IOs, 8870206464 bytes 2026-02-22T22:08:55.095 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20200 IOs, 8928830464 bytes 2026-02-22T22:08:55.154 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20300 IOs, 8990188544 bytes 2026-02-22T22:08:55.202 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20400 IOs, 9036220416 bytes 2026-02-22T22:08:55.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20500 IOs, 9102653440 bytes 2026-02-22T22:08:55.305 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20600 IOs, 9140612096 bytes 2026-02-22T22:08:55.361 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20700 IOs, 9204749312 bytes 2026-02-22T22:08:55.395 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20800 IOs, 9216864256 bytes 2026-02-22T22:08:55.437 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20900 IOs, 9241763840 bytes 2026-02-22T22:08:55.478 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21000 IOs, 9268692992 bytes 2026-02-22T22:08:55.539 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21100 IOs, 9350736896 bytes 2026-02-22T22:08:55.585 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21200 IOs, 9375022080 bytes 2026-02-22T22:08:55.637 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21300 IOs, 9422140416 bytes 2026-02-22T22:08:55.665 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21400 IOs, 9422757888 bytes 2026-02-22T22:08:55.730 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21500 IOs, 9509699584 bytes 2026-02-22T22:08:55.788 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21600 IOs, 9571228672 bytes 2026-02-22T22:08:55.847 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21700 IOs, 9642121216 bytes 2026-02-22T22:08:55.900 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21800 IOs, 9702681600 bytes 2026-02-22T22:08:55.945 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21900 IOs, 9751026688 bytes 2026-02-22T22:08:55.995 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22000 IOs, 9792214016 bytes 2026-02-22T22:08:56.026 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22100 IOs, 9797051392 bytes 2026-02-22T22:08:56.083 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22200 IOs, 9865486336 bytes 2026-02-22T22:08:56.130 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22300 IOs, 9900579840 bytes 2026-02-22T22:08:56.177 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22400 IOs, 9923492864 bytes 2026-02-22T22:08:56.220 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22500 IOs, 9948419072 bytes 2026-02-22T22:08:56.262 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22600 IOs, 9979888640 bytes 2026-02-22T22:08:56.286 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ error_code=124 2026-02-22T22:08:56.287 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set -e 2026-02-22T22:08:56.287 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' 124 -eq 124 ']' 2026-02-22T22:08:56.287 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:08:56.287 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_present cluster1 mirror image_6 present 2026-02-22T22:08:56.287 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:08:56.287 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:08:56.287 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_6 2026-02-22T22:08:56.287 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=present 2026-02-22T22:08:56.287 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:08:56.287 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:08:56.287 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:08:56.288 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_6 2026-02-22T22:08:56.288 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:08:56.288 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:08:56.288 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_6 2026-02-22T22:08:56.288 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_6 2026-02-22T22:08:56.288 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:08:56.315 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image_id=126082022170 2026-02-22T22:08:56.315 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:08:56.316 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.1 2026-02-22T22:08:56.417 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_present cluster1 mirror image_6 present 126082022170 2026-02-22T22:08:56.417 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:08:56.417 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:08:56.417 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_6 2026-02-22T22:08:56.417 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=present 2026-02-22T22:08:56.417 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id=126082022170 2026-02-22T22:08:56.417 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=deleted 2026-02-22T22:08:56.417 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_image_id 2026-02-22T22:08:56.417 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_6 2026-02-22T22:08:56.418 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:08:56.418 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:08:56.418 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_6 2026-02-22T22:08:56.418 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_6 2026-02-22T22:08:56.418 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:08:56.444 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_image_id=126082022170 2026-02-22T22:08:56.444 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n 126082022170 2026-02-22T22:08:56.444 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -z 126082022170 -o 126082022170 = 126082022170 2026-02-22T22:08:56.444 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_state=present 2026-02-22T22:08:56.445 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test present = present 2026-02-22T22:08:56.445 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:08:56.445 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ disable_mirror cluster2 mirror image_6 2026-02-22T22:08:56.445 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:08:56.445 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:08:56.445 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_6 2026-02-22T22:08:56.445 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_6 2026-02-22T22:08:57.290 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial026.stderr:2026-02-22T22:08:57.288+0000 7f1baf63e640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x5647246ffc00 scan_remote_mirror_snapshots: failed to locate remote start snapshot: snap_id=190 2026-02-22T22:08:57.290 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial026.stderr:2026-02-22T22:08:57.288+0000 7f1baf63e640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x5647246ffc00 scan_remote_mirror_snapshots: split-brain detected: failed to find matching non-primary snapshot in remote image: local_snap_id_start=190, local_snap_ns=[mirror state=non-primary, complete=1, mirror_peer_uuids=, primary_mirror_uuid=cfa81703-817a-4074-adbc-5e6e200790bf, primary_snap_id=be, last_copied_object_number=128, snap_seqs={190=18446744073709551614}] 2026-02-22T22:08:59.095 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.9160409594331682) > weight (0.25) 2026-02-22T22:08:59.095 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.7441614192848325) > weight (0.25) 2026-02-22T22:08:59.095 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.2552845898647126) > weight (0.25) 2026-02-22T22:08:59.095 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.3 with skip (0.574969378180241) > weight (0.25) 2026-02-22T22:08:59.096 INFO:tasks.rbd_mirror_thrash:waiting for 76.33957765394423 secs before thrashing 2026-02-22T22:09:02.293 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Mirroring disabled 2026-02-22T22:09:02.299 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_present cluster1 mirror image_6 deleted 2026-02-22T22:09:02.300 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:09:02.300 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:09:02.300 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_6 2026-02-22T22:09:02.300 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=deleted 2026-02-22T22:09:02.300 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:09:02.300 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:09:02.300 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:09:02.300 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_6 2026-02-22T22:09:02.300 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:09:02.300 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:09:02.301 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_6 2026-02-22T22:09:02.301 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_6 2026-02-22T22:09:02.301 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:09:02.322 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_6: (2) No such file or directory 2026-02-22T22:09:02.324 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image_id= 2026-02-22T22:09:02.324 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:09:02.325 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.1 2026-02-22T22:09:02.425 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_present cluster1 mirror image_6 deleted '' 2026-02-22T22:09:02.426 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:09:02.426 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:09:02.426 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_6 2026-02-22T22:09:02.426 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=deleted 2026-02-22T22:09:02.426 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:09:02.426 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=deleted 2026-02-22T22:09:02.426 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_image_id 2026-02-22T22:09:02.426 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_6 2026-02-22T22:09:02.426 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:09:02.426 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:09:02.426 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_6 2026-02-22T22:09:02.426 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_6 2026-02-22T22:09:02.426 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:09:02.448 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_6: (2) No such file or directory 2026-02-22T22:09:02.450 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_image_id= 2026-02-22T22:09:02.450 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:09:02.450 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test deleted = deleted 2026-02-22T22:09:02.450 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:09:02.450 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ purge_snapshots cluster2 mirror image_6 2026-02-22T22:09:02.450 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:09:02.450 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:09:02.450 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_6 2026-02-22T22:09:02.450 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap purge mirror/image_6 2026-02-22T22:09:02.480 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image_retry cluster2 mirror image_6 2026-02-22T22:09:02.480 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:09:02.481 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:09:02.481 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_6 2026-02-22T22:09:02.481 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:09:02.481 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0 2026-02-22T22:09:02.482 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image cluster2 mirror image_6 2026-02-22T22:09:02.482 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:09:02.482 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:09:02.482 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_6 2026-02-22T22:09:02.482 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_6 2026-02-22T22:09:02.510 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 rm mirror/image_6 2026-02-22T22:09:26.545 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2026-02-22T22:09:26.549 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:09:26.549 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 10` 2026-02-22T22:09:26.549 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_7 2026-02-22T22:09:26.549 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image cluster2 mirror image_7 512M 2026-02-22T22:09:26.549 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:09:26.549 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T22:09:26.549 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:09:26.549 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T22:09:26.550 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_7 2026-02-22T22:09:26.550 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T22:09:26.550 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local size=128 2026-02-22T22:09:26.550 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n 512M ']' 2026-02-22T22:09:26.550 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ size=512M 2026-02-22T22:09:26.550 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T22:09:26.550 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering,exclusive-lock mirror/image_7 2026-02-22T22:09:26.581 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ enable_mirror cluster2 mirror image_7 2026-02-22T22:09:26.581 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:09:26.581 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:09:26.581 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_7 2026-02-22T22:09:26.581 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T22:09:26.581 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_7 snapshot 2026-02-22T22:09:27.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Mirroring enabled 2026-02-22T22:09:27.258 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 info mirror/image_7 2026-02-22T22:09:27.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_7': 2026-02-22T22:09:27.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 512 MiB in 128 objects 2026-02-22T22:09:27.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T22:09:27.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 1 2026-02-22T22:09:27.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 1969c4967438 2026-02-22T22:09:27.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.1969c4967438 2026-02-22T22:09:27.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T22:09:27.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock 2026-02-22T22:09:27.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T22:09:27.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T22:09:27.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 22:09:26 2026 2026-02-22T22:09:27.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 22:09:26 2026 2026-02-22T22:09:27.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 22:09:26 2026 2026-02-22T22:09:27.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T22:09:27.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T22:09:27.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: 68812cd5-d9ab-4fed-aecc-bccdd26591c2 2026-02-22T22:09:27.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: true 2026-02-22T22:09:27.285 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ stress_write_image cluster2 mirror image_7 2026-02-22T22:09:27.286 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:09:27.286 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:09:27.286 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_7 2026-02-22T22:09:27.286 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-22T22:09:27.287 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local duration=14 2026-02-22T22:09:27.287 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set +e 2026-02-22T22:09:27.287 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ timeout 14s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_7 --debug-rbd=20 --debug-journaler=20 2026-02-22T22:09:27.381 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:100 IOs, 71923712 bytes 2026-02-22T22:09:27.426 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:200 IOs, 110904320 bytes 2026-02-22T22:09:27.483 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:300 IOs, 179799040 bytes 2026-02-22T22:09:27.520 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:400 IOs, 205455360 bytes 2026-02-22T22:09:27.562 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:500 IOs, 241891328 bytes 2026-02-22T22:09:27.611 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:600 IOs, 293818368 bytes 2026-02-22T22:09:27.649 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:700 IOs, 329991168 bytes 2026-02-22T22:09:27.686 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:800 IOs, 336561152 bytes 2026-02-22T22:09:27.717 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:900 IOs, 356050944 bytes 2026-02-22T22:09:27.770 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1000 IOs, 406378496 bytes 2026-02-22T22:09:27.815 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1100 IOs, 445405184 bytes 2026-02-22T22:09:27.880 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1200 IOs, 542815232 bytes 2026-02-22T22:09:27.912 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1300 IOs, 548848640 bytes 2026-02-22T22:09:27.952 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1400 IOs, 574980096 bytes 2026-02-22T22:09:28.002 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1500 IOs, 631515136 bytes 2026-02-22T22:09:28.062 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1600 IOs, 719682560 bytes 2026-02-22T22:09:28.122 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1700 IOs, 798387200 bytes 2026-02-22T22:09:28.156 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1800 IOs, 807937024 bytes 2026-02-22T22:09:28.195 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1900 IOs, 826506240 bytes 2026-02-22T22:09:28.238 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2000 IOs, 863044608 bytes 2026-02-22T22:09:28.316 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2100 IOs, 991547392 bytes 2026-02-22T22:09:28.356 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2200 IOs, 1012202496 bytes 2026-02-22T22:09:28.404 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2300 IOs, 1063667712 bytes 2026-02-22T22:09:28.445 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2400 IOs, 1089699840 bytes 2026-02-22T22:09:28.484 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2500 IOs, 1115467776 bytes 2026-02-22T22:09:28.528 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2600 IOs, 1164267520 bytes 2026-02-22T22:09:28.569 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2700 IOs, 1183188992 bytes 2026-02-22T22:09:28.617 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2800 IOs, 1234237440 bytes 2026-02-22T22:09:28.644 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2900 IOs, 1234861056 bytes 2026-02-22T22:09:28.684 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3000 IOs, 1294007296 bytes 2026-02-22T22:09:28.734 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3100 IOs, 1317811200 bytes 2026-02-22T22:09:28.787 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3200 IOs, 1382149120 bytes 2026-02-22T22:09:28.833 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3300 IOs, 1422274560 bytes 2026-02-22T22:09:28.886 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3400 IOs, 1492631552 bytes 2026-02-22T22:09:28.938 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3500 IOs, 1557014528 bytes 2026-02-22T22:09:28.982 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3600 IOs, 1593560064 bytes 2026-02-22T22:09:29.032 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3700 IOs, 1649492992 bytes 2026-02-22T22:09:29.082 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3800 IOs, 1715108864 bytes 2026-02-22T22:09:29.120 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3900 IOs, 1737003008 bytes 2026-02-22T22:09:29.176 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4000 IOs, 1802202112 bytes 2026-02-22T22:09:29.237 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4100 IOs, 1888977920 bytes 2026-02-22T22:09:29.301 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4200 IOs, 1954608128 bytes 2026-02-22T22:09:29.334 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4300 IOs, 1957011456 bytes 2026-02-22T22:09:29.366 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4400 IOs, 1965924352 bytes 2026-02-22T22:09:29.405 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4500 IOs, 1995301888 bytes 2026-02-22T22:09:29.433 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4600 IOs, 1995919360 bytes 2026-02-22T22:09:29.490 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4700 IOs, 2067579904 bytes 2026-02-22T22:09:29.530 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4800 IOs, 2084866048 bytes 2026-02-22T22:09:29.576 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4900 IOs, 2132782080 bytes 2026-02-22T22:09:29.625 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5000 IOs, 2176513024 bytes 2026-02-22T22:09:29.668 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5100 IOs, 2212485120 bytes 2026-02-22T22:09:29.728 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5200 IOs, 2301274112 bytes 2026-02-22T22:09:29.779 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5300 IOs, 2357337088 bytes 2026-02-22T22:09:29.830 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5400 IOs, 2405791744 bytes 2026-02-22T22:09:29.885 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5500 IOs, 2478716928 bytes 2026-02-22T22:09:29.934 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5600 IOs, 2533627904 bytes 2026-02-22T22:09:29.980 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5700 IOs, 2578028544 bytes 2026-02-22T22:09:30.023 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5800 IOs, 2613089280 bytes 2026-02-22T22:09:30.087 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5900 IOs, 2705327104 bytes 2026-02-22T22:09:30.141 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6000 IOs, 2767233024 bytes 2026-02-22T22:09:30.192 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6100 IOs, 2819619840 bytes 2026-02-22T22:09:30.425 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6200 IOs, 2829807616 bytes 2026-02-22T22:09:30.468 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6300 IOs, 2861495296 bytes 2026-02-22T22:09:30.527 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6400 IOs, 2941802496 bytes 2026-02-22T22:09:30.580 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6500 IOs, 3026830336 bytes 2026-02-22T22:09:30.637 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6600 IOs, 3078594560 bytes 2026-02-22T22:09:30.666 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6700 IOs, 3079217152 bytes 2026-02-22T22:09:30.709 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6800 IOs, 3114086400 bytes 2026-02-22T22:09:30.763 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6900 IOs, 3186836480 bytes 2026-02-22T22:09:30.811 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7000 IOs, 3229889536 bytes 2026-02-22T22:09:30.855 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7100 IOs, 3267122176 bytes 2026-02-22T22:09:30.890 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7200 IOs, 3285751808 bytes 2026-02-22T22:09:30.945 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7300 IOs, 3329848320 bytes 2026-02-22T22:09:31.006 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7400 IOs, 3418316800 bytes 2026-02-22T22:09:31.050 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7500 IOs, 3446874112 bytes 2026-02-22T22:09:31.085 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7600 IOs, 3460111360 bytes 2026-02-22T22:09:31.125 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7700 IOs, 3492362240 bytes 2026-02-22T22:09:31.180 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7800 IOs, 3565107200 bytes 2026-02-22T22:09:31.230 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7900 IOs, 3610118144 bytes 2026-02-22T22:09:31.270 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8000 IOs, 3630979072 bytes 2026-02-22T22:09:31.300 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8100 IOs, 3631596544 bytes 2026-02-22T22:09:31.346 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8200 IOs, 3675799552 bytes 2026-02-22T22:09:31.384 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8300 IOs, 3693372416 bytes 2026-02-22T22:09:31.447 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8400 IOs, 3782624256 bytes 2026-02-22T22:09:31.503 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8500 IOs, 3851401216 bytes 2026-02-22T22:09:31.541 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8600 IOs, 3878426624 bytes 2026-02-22T22:09:31.585 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8700 IOs, 3919763456 bytes 2026-02-22T22:09:31.625 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8800 IOs, 3945552896 bytes 2026-02-22T22:09:31.685 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8900 IOs, 4025150464 bytes 2026-02-22T22:09:31.715 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9000 IOs, 4025766912 bytes 2026-02-22T22:09:31.781 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9100 IOs, 4120199168 bytes 2026-02-22T22:09:31.832 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9200 IOs, 4155742208 bytes 2026-02-22T22:09:31.892 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9300 IOs, 4235144192 bytes 2026-02-22T22:09:31.941 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9400 IOs, 4293797888 bytes 2026-02-22T22:09:31.987 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9500 IOs, 4324366336 bytes 2026-02-22T22:09:32.042 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9600 IOs, 4383391744 bytes 2026-02-22T22:09:32.090 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9700 IOs, 4429360128 bytes 2026-02-22T22:09:32.146 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9800 IOs, 4506096640 bytes 2026-02-22T22:09:32.185 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9900 IOs, 4536469504 bytes 2026-02-22T22:09:32.252 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10000 IOs, 4627050496 bytes 2026-02-22T22:09:32.303 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10100 IOs, 4669352960 bytes 2026-02-22T22:09:32.340 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10200 IOs, 4698510336 bytes 2026-02-22T22:09:32.396 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10300 IOs, 4753853440 bytes 2026-02-22T22:09:32.437 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10400 IOs, 4773117952 bytes 2026-02-22T22:09:32.486 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10500 IOs, 4818575360 bytes 2026-02-22T22:09:32.532 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10600 IOs, 4862297088 bytes 2026-02-22T22:09:32.575 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10700 IOs, 4902665216 bytes 2026-02-22T22:09:32.622 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10800 IOs, 4943074304 bytes 2026-02-22T22:09:32.654 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10900 IOs, 4946656256 bytes 2026-02-22T22:09:32.702 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11000 IOs, 4992031744 bytes 2026-02-22T22:09:32.760 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11100 IOs, 5064199168 bytes 2026-02-22T22:09:32.796 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11200 IOs, 5087910912 bytes 2026-02-22T22:09:32.844 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11300 IOs, 5132242944 bytes 2026-02-22T22:09:32.886 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11400 IOs, 5163625472 bytes 2026-02-22T22:09:32.929 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11500 IOs, 5216188416 bytes 2026-02-22T22:09:32.975 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11600 IOs, 5256614912 bytes 2026-02-22T22:09:33.026 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11700 IOs, 5310572544 bytes 2026-02-22T22:09:33.072 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11800 IOs, 5344918528 bytes 2026-02-22T22:09:33.123 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:11900 IOs, 5402897408 bytes 2026-02-22T22:09:33.163 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12000 IOs, 5453124608 bytes 2026-02-22T22:09:33.233 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12100 IOs, 5527261184 bytes 2026-02-22T22:09:33.279 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12200 IOs, 5566905344 bytes 2026-02-22T22:09:33.325 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12300 IOs, 5593017344 bytes 2026-02-22T22:09:33.392 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12400 IOs, 5692577792 bytes 2026-02-22T22:09:33.453 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12500 IOs, 5757387776 bytes 2026-02-22T22:09:33.517 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12600 IOs, 5850193920 bytes 2026-02-22T22:09:33.564 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12700 IOs, 5880732672 bytes 2026-02-22T22:09:33.610 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12800 IOs, 5922250752 bytes 2026-02-22T22:09:33.668 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:12900 IOs, 6009719808 bytes 2026-02-22T22:09:33.713 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13000 IOs, 6025526272 bytes 2026-02-22T22:09:33.765 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13100 IOs, 6063636480 bytes 2026-02-22T22:09:33.807 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13200 IOs, 6089921536 bytes 2026-02-22T22:09:33.860 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13300 IOs, 6117950464 bytes 2026-02-22T22:09:33.914 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13400 IOs, 6157996032 bytes 2026-02-22T22:09:33.955 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13500 IOs, 6171650048 bytes 2026-02-22T22:09:34.016 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13600 IOs, 6231448576 bytes 2026-02-22T22:09:34.059 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13700 IOs, 6246459392 bytes 2026-02-22T22:09:34.114 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13800 IOs, 6290207744 bytes 2026-02-22T22:09:34.157 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:13900 IOs, 6307271680 bytes 2026-02-22T22:09:34.214 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14000 IOs, 6347714560 bytes 2026-02-22T22:09:34.272 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14100 IOs, 6388508672 bytes 2026-02-22T22:09:34.354 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14200 IOs, 6432753664 bytes 2026-02-22T22:09:34.401 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14300 IOs, 6454273024 bytes 2026-02-22T22:09:34.452 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14400 IOs, 6493134848 bytes 2026-02-22T22:09:34.514 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14500 IOs, 6574700544 bytes 2026-02-22T22:09:34.555 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14600 IOs, 6578857984 bytes 2026-02-22T22:09:34.611 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14700 IOs, 6632839168 bytes 2026-02-22T22:09:34.653 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14800 IOs, 6651046912 bytes 2026-02-22T22:09:34.693 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:14900 IOs, 6663604224 bytes 2026-02-22T22:09:34.734 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15000 IOs, 6675514368 bytes 2026-02-22T22:09:34.783 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15100 IOs, 6705331200 bytes 2026-02-22T22:09:34.831 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15200 IOs, 6744288256 bytes 2026-02-22T22:09:34.882 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15300 IOs, 6790805504 bytes 2026-02-22T22:09:34.929 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15400 IOs, 6815211520 bytes 2026-02-22T22:09:34.977 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15500 IOs, 6838652928 bytes 2026-02-22T22:09:35.033 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15600 IOs, 6892997632 bytes 2026-02-22T22:09:35.068 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15700 IOs, 6893599744 bytes 2026-02-22T22:09:35.123 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15800 IOs, 6935832576 bytes 2026-02-22T22:09:35.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:15900 IOs, 6959860736 bytes 2026-02-22T22:09:35.218 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16000 IOs, 6997568512 bytes 2026-02-22T22:09:35.276 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16100 IOs, 7042686976 bytes 2026-02-22T22:09:35.330 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16200 IOs, 7060950016 bytes 2026-02-22T22:09:35.381 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16300 IOs, 7089617920 bytes 2026-02-22T22:09:35.427 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16400 IOs, 7113513984 bytes 2026-02-22T22:09:35.485 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16500 IOs, 7159152640 bytes 2026-02-22T22:09:35.529 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16600 IOs, 7184803840 bytes 2026-02-22T22:09:35.569 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16700 IOs, 7196690432 bytes 2026-02-22T22:09:35.612 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16800 IOs, 7244684288 bytes 2026-02-22T22:09:35.657 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:16900 IOs, 7250971648 bytes 2026-02-22T22:09:35.736 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17000 IOs, 7369633792 bytes 2026-02-22T22:09:35.790 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17100 IOs, 7411005440 bytes 2026-02-22T22:09:35.833 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17200 IOs, 7435999232 bytes 2026-02-22T22:09:35.876 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17300 IOs, 7449039872 bytes 2026-02-22T22:09:35.934 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17400 IOs, 7517175808 bytes 2026-02-22T22:09:35.986 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17500 IOs, 7542584320 bytes 2026-02-22T22:09:36.063 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17600 IOs, 7654807552 bytes 2026-02-22T22:09:36.103 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17700 IOs, 7663434752 bytes 2026-02-22T22:09:36.152 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17800 IOs, 7692813312 bytes 2026-02-22T22:09:36.223 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:17900 IOs, 7795586048 bytes 2026-02-22T22:09:36.287 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18000 IOs, 7848226816 bytes 2026-02-22T22:09:36.329 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18100 IOs, 7861961728 bytes 2026-02-22T22:09:36.370 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18200 IOs, 7863886848 bytes 2026-02-22T22:09:36.420 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18300 IOs, 7903643648 bytes 2026-02-22T22:09:36.478 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18400 IOs, 7960440832 bytes 2026-02-22T22:09:36.519 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18500 IOs, 7978813440 bytes 2026-02-22T22:09:36.572 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18600 IOs, 8022724608 bytes 2026-02-22T22:09:36.647 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18700 IOs, 8117696512 bytes 2026-02-22T22:09:36.708 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18800 IOs, 8175336448 bytes 2026-02-22T22:09:36.763 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:18900 IOs, 8223576064 bytes 2026-02-22T22:09:36.803 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19000 IOs, 8232384512 bytes 2026-02-22T22:09:36.853 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19100 IOs, 8294936576 bytes 2026-02-22T22:09:36.939 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19200 IOs, 8398862336 bytes 2026-02-22T22:09:36.976 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19300 IOs, 8399450112 bytes 2026-02-22T22:09:37.031 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19400 IOs, 8448043008 bytes 2026-02-22T22:09:37.080 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19500 IOs, 8501157888 bytes 2026-02-22T22:09:37.134 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19600 IOs, 8536212480 bytes 2026-02-22T22:09:37.184 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19700 IOs, 8574241792 bytes 2026-02-22T22:09:37.250 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19800 IOs, 8682593280 bytes 2026-02-22T22:09:37.323 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:19900 IOs, 8760861696 bytes 2026-02-22T22:09:37.378 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20000 IOs, 8788608000 bytes 2026-02-22T22:09:37.419 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20100 IOs, 8801304576 bytes 2026-02-22T22:09:37.463 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20200 IOs, 8817052672 bytes 2026-02-22T22:09:37.510 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20300 IOs, 8850209792 bytes 2026-02-22T22:09:37.568 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20400 IOs, 8924545024 bytes 2026-02-22T22:09:37.612 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20500 IOs, 8929868800 bytes 2026-02-22T22:09:37.671 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20600 IOs, 8991962112 bytes 2026-02-22T22:09:37.716 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20700 IOs, 9011203072 bytes 2026-02-22T22:09:37.775 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20800 IOs, 9059866624 bytes 2026-02-22T22:09:37.833 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:20900 IOs, 9121550336 bytes 2026-02-22T22:09:37.879 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21000 IOs, 9136405504 bytes 2026-02-22T22:09:37.939 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21100 IOs, 9194281984 bytes 2026-02-22T22:09:37.981 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21200 IOs, 9214512128 bytes 2026-02-22T22:09:38.046 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21300 IOs, 9277926400 bytes 2026-02-22T22:09:38.108 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21400 IOs, 9349153792 bytes 2026-02-22T22:09:38.144 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21500 IOs, 9358841856 bytes 2026-02-22T22:09:38.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21600 IOs, 9412049920 bytes 2026-02-22T22:09:38.239 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21700 IOs, 9422991360 bytes 2026-02-22T22:09:38.296 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21800 IOs, 9470771200 bytes 2026-02-22T22:09:38.342 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:21900 IOs, 9498554368 bytes 2026-02-22T22:09:38.392 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22000 IOs, 9515484160 bytes 2026-02-22T22:09:38.439 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22100 IOs, 9539099648 bytes 2026-02-22T22:09:38.472 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22200 IOs, 9539705856 bytes 2026-02-22T22:09:38.512 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22300 IOs, 9554031616 bytes 2026-02-22T22:09:38.568 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22400 IOs, 9619268608 bytes 2026-02-22T22:09:38.630 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22500 IOs, 9670063104 bytes 2026-02-22T22:09:38.665 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22600 IOs, 9691957248 bytes 2026-02-22T22:09:38.743 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22700 IOs, 9770475520 bytes 2026-02-22T22:09:38.781 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22800 IOs, 9776953344 bytes 2026-02-22T22:09:38.829 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:22900 IOs, 9806529536 bytes 2026-02-22T22:09:38.892 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23000 IOs, 9873683456 bytes 2026-02-22T22:09:38.939 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23100 IOs, 9916734464 bytes 2026-02-22T22:09:38.987 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23200 IOs, 9953051648 bytes 2026-02-22T22:09:39.047 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23300 IOs, 10032536576 bytes 2026-02-22T22:09:39.097 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23400 IOs, 10053729280 bytes 2026-02-22T22:09:39.147 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23500 IOs, 10087108608 bytes 2026-02-22T22:09:39.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23600 IOs, 10120269824 bytes 2026-02-22T22:09:39.238 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23700 IOs, 10160464896 bytes 2026-02-22T22:09:39.281 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23800 IOs, 10179319808 bytes 2026-02-22T22:09:39.345 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:23900 IOs, 10258495488 bytes 2026-02-22T22:09:39.390 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24000 IOs, 10267021312 bytes 2026-02-22T22:09:39.435 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24100 IOs, 10289323008 bytes 2026-02-22T22:09:39.485 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24200 IOs, 10319644672 bytes 2026-02-22T22:09:39.534 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24300 IOs, 10357777408 bytes 2026-02-22T22:09:39.602 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24400 IOs, 10440739840 bytes 2026-02-22T22:09:39.634 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24500 IOs, 10441350144 bytes 2026-02-22T22:09:39.692 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24600 IOs, 10501408768 bytes 2026-02-22T22:09:39.733 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24700 IOs, 10513168384 bytes 2026-02-22T22:09:39.791 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24800 IOs, 10574492672 bytes 2026-02-22T22:09:39.844 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:24900 IOs, 10614071296 bytes 2026-02-22T22:09:39.898 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25000 IOs, 10664772608 bytes 2026-02-22T22:09:39.942 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25100 IOs, 10688381952 bytes 2026-02-22T22:09:39.983 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25200 IOs, 10709918720 bytes 2026-02-22T22:09:40.021 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25300 IOs, 10720636928 bytes 2026-02-22T22:09:40.072 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25400 IOs, 10771315712 bytes 2026-02-22T22:09:40.126 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25500 IOs, 10810710016 bytes 2026-02-22T22:09:40.175 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25600 IOs, 10844546048 bytes 2026-02-22T22:09:40.241 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25700 IOs, 10920213504 bytes 2026-02-22T22:09:40.313 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25800 IOs, 11008958464 bytes 2026-02-22T22:09:40.362 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:25900 IOs, 11040687104 bytes 2026-02-22T22:09:40.412 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:26000 IOs, 11046651904 bytes 2026-02-22T22:09:40.457 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:26100 IOs, 11066499072 bytes 2026-02-22T22:09:40.511 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:26200 IOs, 11114136576 bytes 2026-02-22T22:09:40.563 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:26300 IOs, 11149257728 bytes 2026-02-22T22:09:40.615 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:26400 IOs, 11186064384 bytes 2026-02-22T22:09:40.651 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:26500 IOs, 11189094400 bytes 2026-02-22T22:09:40.691 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:26600 IOs, 11194635264 bytes 2026-02-22T22:09:40.734 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:26700 IOs, 11219970048 bytes 2026-02-22T22:09:40.777 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:26800 IOs, 11236740096 bytes 2026-02-22T22:09:40.834 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:26900 IOs, 11305756672 bytes 2026-02-22T22:09:40.889 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:27000 IOs, 11356598272 bytes 2026-02-22T22:09:40.940 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:27100 IOs, 11404846080 bytes 2026-02-22T22:09:40.989 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:27200 IOs, 11451517952 bytes 2026-02-22T22:09:41.044 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:27300 IOs, 11479361536 bytes 2026-02-22T22:09:41.095 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:27400 IOs, 11510303744 bytes 2026-02-22T22:09:41.149 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:27500 IOs, 11591318528 bytes 2026-02-22T22:09:41.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:27600 IOs, 11625751552 bytes 2026-02-22T22:09:41.248 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:27700 IOs, 11652405248 bytes 2026-02-22T22:09:41.294 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ error_code=124 2026-02-22T22:09:41.294 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set -e 2026-02-22T22:09:41.295 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' 124 -eq 124 ']' 2026-02-22T22:09:41.295 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:09:41.295 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_present cluster1 mirror image_7 present 2026-02-22T22:09:41.295 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:09:41.295 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:09:41.295 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_7 2026-02-22T22:09:41.295 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=present 2026-02-22T22:09:41.295 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:09:41.295 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:09:41.295 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:09:41.295 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_7 2026-02-22T22:09:41.296 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:09:41.296 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:09:41.296 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_7 2026-02-22T22:09:41.296 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_7 2026-02-22T22:09:41.296 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:09:41.324 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image_id=126094859fd6 2026-02-22T22:09:41.325 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:09:41.325 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.1 2026-02-22T22:09:41.426 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_present cluster1 mirror image_7 present 126094859fd6 2026-02-22T22:09:41.426 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:09:41.426 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:09:41.426 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_7 2026-02-22T22:09:41.426 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=present 2026-02-22T22:09:41.426 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id=126094859fd6 2026-02-22T22:09:41.427 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=deleted 2026-02-22T22:09:41.427 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_image_id 2026-02-22T22:09:41.427 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_7 2026-02-22T22:09:41.427 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:09:41.427 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:09:41.427 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_7 2026-02-22T22:09:41.427 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_7 2026-02-22T22:09:41.427 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:09:41.455 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_image_id=126094859fd6 2026-02-22T22:09:41.455 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n 126094859fd6 2026-02-22T22:09:41.456 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -z 126094859fd6 -o 126094859fd6 = 126094859fd6 2026-02-22T22:09:41.456 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_state=present 2026-02-22T22:09:41.456 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test present = present 2026-02-22T22:09:41.456 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:09:41.456 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ disable_mirror cluster2 mirror image_7 2026-02-22T22:09:41.456 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:09:41.457 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:09:41.457 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_7 2026-02-22T22:09:41.457 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_7 2026-02-22T22:09:42.237 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial026.stderr:2026-02-22T22:09:42.235+0000 7f1baee3d640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x5647241ea400 scan_remote_mirror_snapshots: failed to locate remote start snapshot: snap_id=191 2026-02-22T22:09:42.237 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial026.stderr:2026-02-22T22:09:42.235+0000 7f1baee3d640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x5647241ea400 scan_remote_mirror_snapshots: split-brain detected: failed to find matching non-primary snapshot in remote image: local_snap_id_start=191, local_snap_ns=[mirror state=non-primary, complete=1, mirror_peer_uuids=, primary_mirror_uuid=cfa81703-817a-4074-adbc-5e6e200790bf, primary_snap_id=bf, last_copied_object_number=128, snap_seqs={191=18446744073709551614}] 2026-02-22T22:09:44.936 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial026.stderr:2026-02-22T22:09:44.934+0000 7f5f98680640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x56395249e4e0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-22T22:09:44.936 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial026.stderr:2026-02-22T22:09:44.934+0000 7f5f9ee8d640 -1 rbd::mirror::ImageReplayer: 0x563952607680 [3/68812cd5-d9ab-4fed-aecc-bccdd26591c2] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-22T22:09:47.238 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Mirroring disabled 2026-02-22T22:09:47.244 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_present cluster1 mirror image_7 deleted 2026-02-22T22:09:47.244 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:09:47.244 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:09:47.244 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_7 2026-02-22T22:09:47.244 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=deleted 2026-02-22T22:09:47.244 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:09:47.245 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:09:47.245 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:09:47.245 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_7 2026-02-22T22:09:47.245 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:09:47.245 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:09:47.245 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_7 2026-02-22T22:09:47.245 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_7 2026-02-22T22:09:47.245 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:09:47.267 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_7: (2) No such file or directory 2026-02-22T22:09:47.268 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image_id= 2026-02-22T22:09:47.268 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:09:47.269 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.1 2026-02-22T22:09:47.369 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_present cluster1 mirror image_7 deleted '' 2026-02-22T22:09:47.369 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:09:47.369 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:09:47.369 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_7 2026-02-22T22:09:47.369 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=deleted 2026-02-22T22:09:47.369 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:09:47.369 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=deleted 2026-02-22T22:09:47.369 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_image_id 2026-02-22T22:09:47.370 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_7 2026-02-22T22:09:47.370 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:09:47.370 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:09:47.370 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_7 2026-02-22T22:09:47.370 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_7 2026-02-22T22:09:47.370 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:09:47.392 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_7: (2) No such file or directory 2026-02-22T22:09:47.394 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_image_id= 2026-02-22T22:09:47.394 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:09:47.395 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test deleted = deleted 2026-02-22T22:09:47.395 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:09:47.395 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ purge_snapshots cluster2 mirror image_7 2026-02-22T22:09:47.395 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:09:47.395 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:09:47.395 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_7 2026-02-22T22:09:47.395 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap purge mirror/image_7 2026-02-22T22:09:47.424 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image_retry cluster2 mirror image_7 2026-02-22T22:09:47.424 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:09:47.424 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:09:47.424 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_7 2026-02-22T22:09:47.424 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:09:47.424 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0 2026-02-22T22:09:47.425 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image cluster2 mirror image_7 2026-02-22T22:09:47.425 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:09:47.425 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:09:47.425 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_7 2026-02-22T22:09:47.425 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_7 2026-02-22T22:09:47.452 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 rm mirror/image_7 2026-02-22T22:10:11.346 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2026-02-22T22:10:11.349 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:10:11.349 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 10` 2026-02-22T22:10:11.349 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_8 2026-02-22T22:10:11.349 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image cluster2 mirror image_8 512M 2026-02-22T22:10:11.350 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:10:11.350 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T22:10:11.350 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:10:11.350 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T22:10:11.350 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_8 2026-02-22T22:10:11.350 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T22:10:11.350 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local size=128 2026-02-22T22:10:11.350 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n 512M ']' 2026-02-22T22:10:11.350 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ size=512M 2026-02-22T22:10:11.350 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T22:10:11.350 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering,exclusive-lock mirror/image_8 2026-02-22T22:10:11.382 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ enable_mirror cluster2 mirror image_8 2026-02-22T22:10:11.382 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:10:11.382 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:10:11.382 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_8 2026-02-22T22:10:11.382 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T22:10:11.382 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_8 snapshot 2026-02-22T22:10:12.254 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Mirroring enabled 2026-02-22T22:10:12.259 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 info mirror/image_8 2026-02-22T22:10:12.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_8': 2026-02-22T22:10:12.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 512 MiB in 128 objects 2026-02-22T22:10:12.284 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T22:10:12.285 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 1 2026-02-22T22:10:12.285 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 197cd1d235cb 2026-02-22T22:10:12.285 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.197cd1d235cb 2026-02-22T22:10:12.285 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T22:10:12.285 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock 2026-02-22T22:10:12.285 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T22:10:12.285 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T22:10:12.285 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 22:10:11 2026 2026-02-22T22:10:12.285 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 22:10:11 2026 2026-02-22T22:10:12.285 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 22:10:11 2026 2026-02-22T22:10:12.286 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T22:10:12.286 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T22:10:12.286 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: a64713be-a188-4a61-a062-f861d7df3641 2026-02-22T22:10:12.286 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: true 2026-02-22T22:10:12.287 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ stress_write_image cluster2 mirror image_8 2026-02-22T22:10:12.287 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:10:12.287 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:10:12.287 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_8 2026-02-22T22:10:12.287 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-22T22:10:12.288 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local duration=5 2026-02-22T22:10:12.288 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set +e 2026-02-22T22:10:12.288 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ timeout 5s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_8 --debug-rbd=20 --debug-journaler=20 2026-02-22T22:10:12.384 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:100 IOs, 84060160 bytes 2026-02-22T22:10:12.429 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:200 IOs, 122040320 bytes 2026-02-22T22:10:12.478 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:300 IOs, 167634944 bytes 2026-02-22T22:10:12.526 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:400 IOs, 216193024 bytes 2026-02-22T22:10:12.571 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:500 IOs, 251248640 bytes 2026-02-22T22:10:12.622 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:600 IOs, 299051008 bytes 2026-02-22T22:10:12.665 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:700 IOs, 332635136 bytes 2026-02-22T22:10:12.708 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:800 IOs, 360418304 bytes 2026-02-22T22:10:12.765 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:900 IOs, 434270208 bytes 2026-02-22T22:10:12.813 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1000 IOs, 486007808 bytes 2026-02-22T22:10:12.865 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1100 IOs, 554826752 bytes 2026-02-22T22:10:12.906 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1200 IOs, 574202880 bytes 2026-02-22T22:10:12.959 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1300 IOs, 623795200 bytes 2026-02-22T22:10:12.989 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1400 IOs, 624401408 bytes 2026-02-22T22:10:13.047 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1500 IOs, 710288384 bytes 2026-02-22T22:10:13.094 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1600 IOs, 753499136 bytes 2026-02-22T22:10:13.127 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1700 IOs, 761024512 bytes 2026-02-22T22:10:13.162 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1800 IOs, 776370176 bytes 2026-02-22T22:10:13.210 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1900 IOs, 827118592 bytes 2026-02-22T22:10:13.252 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2000 IOs, 874551296 bytes 2026-02-22T22:10:13.323 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2100 IOs, 971990016 bytes 2026-02-22T22:10:13.370 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2200 IOs, 1000171520 bytes 2026-02-22T22:10:13.412 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2300 IOs, 1036081152 bytes 2026-02-22T22:10:13.454 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2400 IOs, 1062428672 bytes 2026-02-22T22:10:13.507 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2500 IOs, 1105528832 bytes 2026-02-22T22:10:13.543 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2600 IOs, 1126343680 bytes 2026-02-22T22:10:13.588 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2700 IOs, 1171463168 bytes 2026-02-22T22:10:13.629 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2800 IOs, 1204511744 bytes 2026-02-22T22:10:13.682 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2900 IOs, 1269977088 bytes 2026-02-22T22:10:13.721 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3000 IOs, 1294488576 bytes 2026-02-22T22:10:13.770 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3100 IOs, 1356768256 bytes 2026-02-22T22:10:13.845 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3200 IOs, 1462340608 bytes 2026-02-22T22:10:13.893 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3300 IOs, 1518007296 bytes 2026-02-22T22:10:13.927 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3400 IOs, 1523304448 bytes 2026-02-22T22:10:13.970 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3500 IOs, 1559456768 bytes 2026-02-22T22:10:14.025 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3600 IOs, 1624878080 bytes 2026-02-22T22:10:14.066 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3700 IOs, 1661314048 bytes 2026-02-22T22:10:14.122 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3800 IOs, 1729946624 bytes 2026-02-22T22:10:14.157 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3900 IOs, 1730584576 bytes 2026-02-22T22:10:14.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4000 IOs, 1748968448 bytes 2026-02-22T22:10:14.230 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4100 IOs, 1766418432 bytes 2026-02-22T22:10:14.270 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4200 IOs, 1788751872 bytes 2026-02-22T22:10:14.306 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4300 IOs, 1808098304 bytes 2026-02-22T22:10:14.377 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4400 IOs, 1914782720 bytes 2026-02-22T22:10:14.412 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4500 IOs, 1918864384 bytes 2026-02-22T22:10:14.458 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4600 IOs, 1964786688 bytes 2026-02-22T22:10:14.533 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4700 IOs, 2050235392 bytes 2026-02-22T22:10:14.566 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4800 IOs, 2058972160 bytes 2026-02-22T22:10:14.613 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4900 IOs, 2092413952 bytes 2026-02-22T22:10:14.651 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5000 IOs, 2117075968 bytes 2026-02-22T22:10:14.699 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5100 IOs, 2168897536 bytes 2026-02-22T22:10:14.734 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5200 IOs, 2183520256 bytes 2026-02-22T22:10:14.775 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5300 IOs, 2210287616 bytes 2026-02-22T22:10:14.837 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5400 IOs, 2286120960 bytes 2026-02-22T22:10:14.878 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5500 IOs, 2312983552 bytes 2026-02-22T22:10:14.924 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5600 IOs, 2346549248 bytes 2026-02-22T22:10:14.967 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5700 IOs, 2369472512 bytes 2026-02-22T22:10:15.007 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5800 IOs, 2398200832 bytes 2026-02-22T22:10:15.039 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5900 IOs, 2398823424 bytes 2026-02-22T22:10:15.100 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6000 IOs, 2474289152 bytes 2026-02-22T22:10:15.152 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6100 IOs, 2531962880 bytes 2026-02-22T22:10:15.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6200 IOs, 2532726784 bytes 2026-02-22T22:10:15.224 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6300 IOs, 2583387136 bytes 2026-02-22T22:10:15.268 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6400 IOs, 2623129600 bytes 2026-02-22T22:10:15.323 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6500 IOs, 2689210368 bytes 2026-02-22T22:10:15.366 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6600 IOs, 2716640256 bytes 2026-02-22T22:10:15.413 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6700 IOs, 2744066048 bytes 2026-02-22T22:10:15.436 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.7555969492267541) > weight (0.25) 2026-02-22T22:10:15.436 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.8570649338121122) > weight (0.25) 2026-02-22T22:10:15.436 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.4887094860876142) > weight (0.25) 2026-02-22T22:10:15.437 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.3 2026-02-22T22:10:15.437 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Sent signal 15 2026-02-22T22:10:15.437 INFO:tasks.rbd_mirror_thrash:waiting for 2.572830266787911 secs before reviving daemons 2026-02-22T22:10:15.438 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial026.stderr:2026-02-22T22:10:15.436+0000 7f1bbce59640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.3 (PID: 60346) UID: 1000 2026-02-22T22:10:15.451 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6800 IOs, 2765359104 bytes 2026-02-22T22:10:15.496 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6900 IOs, 2785106944 bytes 2026-02-22T22:10:15.534 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7000 IOs, 2803028992 bytes 2026-02-22T22:10:15.573 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7100 IOs, 2828673024 bytes 2026-02-22T22:10:15.604 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7200 IOs, 2829292544 bytes 2026-02-22T22:10:15.656 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7300 IOs, 2885699584 bytes 2026-02-22T22:10:15.693 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7400 IOs, 2898496512 bytes 2026-02-22T22:10:15.755 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7500 IOs, 2968297472 bytes 2026-02-22T22:10:15.819 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7600 IOs, 3048682496 bytes 2026-02-22T22:10:15.860 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7700 IOs, 3070175232 bytes 2026-02-22T22:10:15.899 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7800 IOs, 3093881856 bytes 2026-02-22T22:10:15.930 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7900 IOs, 3097791488 bytes 2026-02-22T22:10:15.996 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8000 IOs, 3226593280 bytes 2026-02-22T22:10:16.039 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8100 IOs, 3227197440 bytes 2026-02-22T22:10:16.079 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8200 IOs, 3249614848 bytes 2026-02-22T22:10:16.115 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8300 IOs, 3269156864 bytes 2026-02-22T22:10:16.182 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8400 IOs, 3353134080 bytes 2026-02-22T22:10:16.247 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8500 IOs, 3447225344 bytes 2026-02-22T22:10:16.293 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8600 IOs, 3494097920 bytes 2026-02-22T22:10:16.326 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8700 IOs, 3500264448 bytes 2026-02-22T22:10:16.382 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8800 IOs, 3559588864 bytes 2026-02-22T22:10:16.424 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8900 IOs, 3579305984 bytes 2026-02-22T22:10:16.460 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9000 IOs, 3596640256 bytes 2026-02-22T22:10:16.531 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9100 IOs, 3700142080 bytes 2026-02-22T22:10:16.595 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9200 IOs, 3768521728 bytes 2026-02-22T22:10:16.646 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9300 IOs, 3814437888 bytes 2026-02-22T22:10:16.705 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9400 IOs, 3890342912 bytes 2026-02-22T22:10:16.756 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9500 IOs, 3955380224 bytes 2026-02-22T22:10:16.807 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9600 IOs, 4014051328 bytes 2026-02-22T22:10:16.851 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9700 IOs, 4069183488 bytes 2026-02-22T22:10:16.922 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9800 IOs, 4163510272 bytes 2026-02-22T22:10:16.982 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9900 IOs, 4246376448 bytes 2026-02-22T22:10:17.049 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10000 IOs, 4342333440 bytes 2026-02-22T22:10:17.081 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10100 IOs, 4347337728 bytes 2026-02-22T22:10:17.132 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10200 IOs, 4401771520 bytes 2026-02-22T22:10:17.172 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10300 IOs, 4419624960 bytes 2026-02-22T22:10:17.218 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10400 IOs, 4469338112 bytes 2026-02-22T22:10:17.265 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10500 IOs, 4513207296 bytes 2026-02-22T22:10:17.294 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ error_code=124 2026-02-22T22:10:17.294 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set -e 2026-02-22T22:10:17.295 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' 124 -eq 124 ']' 2026-02-22T22:10:17.295 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:10:17.295 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_present cluster1 mirror image_8 present 2026-02-22T22:10:17.295 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:10:17.295 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:10:17.295 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_8 2026-02-22T22:10:17.295 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=present 2026-02-22T22:10:17.295 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:10:17.295 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:10:17.296 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:10:17.296 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_8 2026-02-22T22:10:17.296 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:10:17.296 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:10:17.296 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_8 2026-02-22T22:10:17.296 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_8 2026-02-22T22:10:17.296 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:10:17.323 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image_id=12601bf06425 2026-02-22T22:10:17.323 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:10:17.323 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.1 2026-02-22T22:10:17.425 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_present cluster1 mirror image_8 present 12601bf06425 2026-02-22T22:10:17.425 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:10:17.425 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:10:17.425 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_8 2026-02-22T22:10:17.425 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=present 2026-02-22T22:10:17.425 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id=12601bf06425 2026-02-22T22:10:17.425 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=deleted 2026-02-22T22:10:17.425 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_image_id 2026-02-22T22:10:17.425 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_8 2026-02-22T22:10:17.426 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:10:17.426 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:10:17.426 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_8 2026-02-22T22:10:17.426 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_8 2026-02-22T22:10:17.426 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:10:17.454 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_image_id=12601bf06425 2026-02-22T22:10:17.454 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n 12601bf06425 2026-02-22T22:10:17.455 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -z 12601bf06425 -o 12601bf06425 = 12601bf06425 2026-02-22T22:10:17.455 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_state=present 2026-02-22T22:10:17.455 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test present = present 2026-02-22T22:10:17.455 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:10:17.455 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ disable_mirror cluster2 mirror image_8 2026-02-22T22:10:17.455 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:10:17.455 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:10:17.455 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_8 2026-02-22T22:10:17.455 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_8 2026-02-22T22:10:18.011 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.3 2026-02-22T22:10:18.011 INFO:teuthology.orchestra.run:waiting for 600 2026-02-22T22:10:18.011 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.3 2026-02-22T22:10:18.012 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Restarting daemon 2026-02-22T22:10:18.012 DEBUG:teuthology.orchestra.run.trial026:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.3 2026-02-22T22:10:18.015 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Started 2026-02-22T22:10:18.015 INFO:tasks.rbd_mirror_thrash:waiting for 82.10234421586242 secs before thrashing 2026-02-22T22:10:23.296 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial026.stderr:2026-02-22T22:10:23.293+0000 7f1e6ed68640 -1 rbd::mirror::ImageMap: 0x55fab60e6000 handle_peer_ack_remove: failed to remove global_image_id=a64713be-a188-4a61-a062-f861d7df3641 2026-02-22T22:10:23.296 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Mirroring disabled 2026-02-22T22:10:23.301 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_present cluster1 mirror image_8 deleted 2026-02-22T22:10:23.301 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:10:23.301 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:10:23.301 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_8 2026-02-22T22:10:23.301 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=deleted 2026-02-22T22:10:23.301 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:10:23.301 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:10:23.301 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:10:23.302 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_8 2026-02-22T22:10:23.302 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:10:23.302 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:10:23.302 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_8 2026-02-22T22:10:23.302 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_8 2026-02-22T22:10:23.302 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:10:23.331 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image_id=12601bf06425 2026-02-22T22:10:23.331 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:10:23.331 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.1 2026-02-22T22:10:23.433 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_present cluster1 mirror image_8 deleted 12601bf06425 2026-02-22T22:10:23.433 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:10:23.434 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:10:23.434 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_8 2026-02-22T22:10:23.434 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=deleted 2026-02-22T22:10:23.434 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id=12601bf06425 2026-02-22T22:10:23.434 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=deleted 2026-02-22T22:10:23.434 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_image_id 2026-02-22T22:10:23.434 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_8 2026-02-22T22:10:23.434 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:10:23.434 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:10:23.434 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_8 2026-02-22T22:10:23.435 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_8 2026-02-22T22:10:23.435 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:10:23.461 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_image_id=12601bf06425 2026-02-22T22:10:23.461 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n 12601bf06425 2026-02-22T22:10:23.461 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -z 12601bf06425 -o 12601bf06425 = 12601bf06425 2026-02-22T22:10:23.461 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_state=present 2026-02-22T22:10:23.462 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test deleted = present 2026-02-22T22:10:23.462 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:10:23.462 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 1 2026-02-22T22:10:24.463 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_present cluster1 mirror image_8 deleted 12601bf06425 2026-02-22T22:10:24.463 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:10:24.463 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:10:24.463 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_8 2026-02-22T22:10:24.463 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=deleted 2026-02-22T22:10:24.464 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id=12601bf06425 2026-02-22T22:10:24.464 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=deleted 2026-02-22T22:10:24.464 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_image_id 2026-02-22T22:10:24.464 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_8 2026-02-22T22:10:24.464 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:10:24.464 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:10:24.464 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_8 2026-02-22T22:10:24.464 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_8 2026-02-22T22:10:24.464 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:10:24.494 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_image_id=12601bf06425 2026-02-22T22:10:24.494 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n 12601bf06425 2026-02-22T22:10:24.494 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -z 12601bf06425 -o 12601bf06425 = 12601bf06425 2026-02-22T22:10:24.495 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_state=present 2026-02-22T22:10:24.495 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test deleted = present 2026-02-22T22:10:24.495 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:10:24.495 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 2 2026-02-22T22:10:26.495 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_present cluster1 mirror image_8 deleted 12601bf06425 2026-02-22T22:10:26.496 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:10:26.496 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:10:26.496 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_8 2026-02-22T22:10:26.496 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=deleted 2026-02-22T22:10:26.496 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id=12601bf06425 2026-02-22T22:10:26.496 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=deleted 2026-02-22T22:10:26.496 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_image_id 2026-02-22T22:10:26.496 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_8 2026-02-22T22:10:26.496 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:10:26.497 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:10:26.497 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_8 2026-02-22T22:10:26.497 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_8 2026-02-22T22:10:26.497 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:10:26.525 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_image_id=12601bf06425 2026-02-22T22:10:26.525 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n 12601bf06425 2026-02-22T22:10:26.525 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -z 12601bf06425 -o 12601bf06425 = 12601bf06425 2026-02-22T22:10:26.525 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_state=present 2026-02-22T22:10:26.526 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test deleted = present 2026-02-22T22:10:26.526 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:10:26.526 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 4 2026-02-22T22:10:28.899 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.0 2026-02-22T22:10:28.899 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Sent signal 15 2026-02-22T22:10:28.899 INFO:tasks.rbd_mirror_thrash:waiting for 4.892151170885909 secs before reviving daemons 2026-02-22T22:10:28.900 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial026.stderr:2026-02-22T22:10:28.898+0000 7f07cb32b640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 (PID: 82058) UID: 1000 2026-02-22T22:10:30.527 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_present cluster1 mirror image_8 deleted 12601bf06425 2026-02-22T22:10:30.527 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:10:30.527 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:10:30.527 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_8 2026-02-22T22:10:30.527 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=deleted 2026-02-22T22:10:30.527 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id=12601bf06425 2026-02-22T22:10:30.528 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=deleted 2026-02-22T22:10:30.528 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_image_id 2026-02-22T22:10:30.528 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_8 2026-02-22T22:10:30.528 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:10:30.528 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:10:30.528 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_8 2026-02-22T22:10:30.528 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_8 2026-02-22T22:10:30.528 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:10:30.559 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_image_id=12601bf06425 2026-02-22T22:10:30.559 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n 12601bf06425 2026-02-22T22:10:30.560 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -z 12601bf06425 -o 12601bf06425 = 12601bf06425 2026-02-22T22:10:30.560 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_state=present 2026-02-22T22:10:30.560 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test deleted = present 2026-02-22T22:10:30.560 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:10:30.560 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 8 2026-02-22T22:10:33.792 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.0 2026-02-22T22:10:33.792 INFO:teuthology.orchestra.run:waiting for 600 2026-02-22T22:10:33.793 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.0 2026-02-22T22:10:33.793 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Restarting daemon 2026-02-22T22:10:33.793 DEBUG:teuthology.orchestra.run.trial026:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 2026-02-22T22:10:33.796 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Started 2026-02-22T22:10:33.796 INFO:tasks.rbd_mirror_thrash:waiting for 118.5882773593167 secs before thrashing 2026-02-22T22:10:38.561 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_present cluster1 mirror image_8 deleted 12601bf06425 2026-02-22T22:10:38.561 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:10:38.561 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:10:38.561 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_8 2026-02-22T22:10:38.561 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=deleted 2026-02-22T22:10:38.561 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id=12601bf06425 2026-02-22T22:10:38.561 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=deleted 2026-02-22T22:10:38.562 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_image_id 2026-02-22T22:10:38.562 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_8 2026-02-22T22:10:38.562 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:10:38.562 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:10:38.562 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_8 2026-02-22T22:10:38.562 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_8 2026-02-22T22:10:38.562 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:10:38.590 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_image_id=12601bf06425 2026-02-22T22:10:38.590 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n 12601bf06425 2026-02-22T22:10:38.590 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -z 12601bf06425 -o 12601bf06425 = 12601bf06425 2026-02-22T22:10:38.590 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_state=present 2026-02-22T22:10:38.590 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test deleted = present 2026-02-22T22:10:38.591 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:10:38.591 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 8 2026-02-22T22:10:46.592 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_present cluster1 mirror image_8 deleted 12601bf06425 2026-02-22T22:10:46.592 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:10:46.592 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:10:46.592 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_8 2026-02-22T22:10:46.593 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=deleted 2026-02-22T22:10:46.593 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id=12601bf06425 2026-02-22T22:10:46.593 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=deleted 2026-02-22T22:10:46.593 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_image_id 2026-02-22T22:10:46.593 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_8 2026-02-22T22:10:46.593 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:10:46.593 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:10:46.593 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_8 2026-02-22T22:10:46.593 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_8 2026-02-22T22:10:46.593 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:10:46.616 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_8: (2) No such file or directory 2026-02-22T22:10:46.618 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_image_id= 2026-02-22T22:10:46.618 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:10:46.619 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test deleted = deleted 2026-02-22T22:10:46.619 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:10:46.619 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ purge_snapshots cluster2 mirror image_8 2026-02-22T22:10:46.619 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:10:46.619 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:10:46.619 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_8 2026-02-22T22:10:46.619 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap purge mirror/image_8 2026-02-22T22:10:46.645 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image_retry cluster2 mirror image_8 2026-02-22T22:10:46.645 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:10:46.646 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:10:46.646 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_8 2026-02-22T22:10:46.646 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:10:46.646 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0 2026-02-22T22:10:46.647 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image cluster2 mirror image_8 2026-02-22T22:10:46.647 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:10:46.647 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:10:46.647 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_8 2026-02-22T22:10:46.647 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_8 2026-02-22T22:10:46.673 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 rm mirror/image_8 2026-02-22T22:10:47.212 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2026-02-22T22:10:47.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:10:47.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 10` 2026-02-22T22:10:47.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_9 2026-02-22T22:10:47.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image cluster2 mirror image_9 512M 2026-02-22T22:10:47.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:10:47.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T22:10:47.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:10:47.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T22:10:47.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_9 2026-02-22T22:10:47.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T22:10:47.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local size=128 2026-02-22T22:10:47.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n 512M ']' 2026-02-22T22:10:47.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ size=512M 2026-02-22T22:10:47.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T22:10:47.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering,exclusive-lock mirror/image_9 2026-02-22T22:10:47.411 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ enable_mirror cluster2 mirror image_9 2026-02-22T22:10:47.411 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:10:47.411 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:10:47.411 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_9 2026-02-22T22:10:47.411 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T22:10:47.411 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_9 snapshot 2026-02-22T22:10:48.128 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Mirroring enabled 2026-02-22T22:10:48.134 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 info mirror/image_9 2026-02-22T22:10:48.159 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_9': 2026-02-22T22:10:48.159 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 512 MiB in 128 objects 2026-02-22T22:10:48.159 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T22:10:48.159 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 1 2026-02-22T22:10:48.159 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 19a292a7f3e 2026-02-22T22:10:48.159 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.19a292a7f3e 2026-02-22T22:10:48.159 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T22:10:48.159 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock 2026-02-22T22:10:48.160 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T22:10:48.160 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T22:10:48.160 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 22:10:47 2026 2026-02-22T22:10:48.160 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 22:10:47 2026 2026-02-22T22:10:48.160 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 22:10:47 2026 2026-02-22T22:10:48.160 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T22:10:48.160 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T22:10:48.160 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: 3e05bd04-5f0d-4827-844d-908f5537362f 2026-02-22T22:10:48.160 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: true 2026-02-22T22:10:48.161 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ stress_write_image cluster2 mirror image_9 2026-02-22T22:10:48.161 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:10:48.161 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:10:48.161 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_9 2026-02-22T22:10:48.162 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-22T22:10:48.162 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local duration=5 2026-02-22T22:10:48.162 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set +e 2026-02-22T22:10:48.162 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ timeout 5s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_9 --debug-rbd=20 --debug-journaler=20 2026-02-22T22:10:48.229 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:100 IOs, 13571072 bytes 2026-02-22T22:10:48.289 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:200 IOs, 80739328 bytes 2026-02-22T22:10:48.332 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:300 IOs, 138948608 bytes 2026-02-22T22:10:48.384 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:400 IOs, 196718592 bytes 2026-02-22T22:10:48.427 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:500 IOs, 234578944 bytes 2026-02-22T22:10:48.457 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:600 IOs, 235182080 bytes 2026-02-22T22:10:48.512 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:700 IOs, 290589696 bytes 2026-02-22T22:10:48.553 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:800 IOs, 320375808 bytes 2026-02-22T22:10:48.606 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:900 IOs, 363102208 bytes 2026-02-22T22:10:48.651 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1000 IOs, 411110400 bytes 2026-02-22T22:10:48.685 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1100 IOs, 419685376 bytes 2026-02-22T22:10:48.734 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1200 IOs, 475446272 bytes 2026-02-22T22:10:48.771 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1300 IOs, 500916224 bytes 2026-02-22T22:10:48.810 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1400 IOs, 527185920 bytes 2026-02-22T22:10:48.861 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1500 IOs, 586400768 bytes 2026-02-22T22:10:48.928 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1600 IOs, 670168064 bytes 2026-02-22T22:10:48.980 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1700 IOs, 741407744 bytes 2026-02-22T22:10:49.051 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1800 IOs, 831777792 bytes 2026-02-22T22:10:49.104 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1900 IOs, 886963200 bytes 2026-02-22T22:10:49.160 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2000 IOs, 950873088 bytes 2026-02-22T22:10:49.205 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2100 IOs, 978422784 bytes 2026-02-22T22:10:49.245 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2200 IOs, 1004315648 bytes 2026-02-22T22:10:49.279 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2300 IOs, 1014087680 bytes 2026-02-22T22:10:49.320 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2400 IOs, 1056565248 bytes 2026-02-22T22:10:49.359 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2500 IOs, 1077876736 bytes 2026-02-22T22:10:49.398 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2600 IOs, 1108458496 bytes 2026-02-22T22:10:49.438 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2700 IOs, 1133320192 bytes 2026-02-22T22:10:49.490 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2800 IOs, 1197367296 bytes 2026-02-22T22:10:49.543 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2900 IOs, 1265480704 bytes 2026-02-22T22:10:49.583 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3000 IOs, 1266093056 bytes 2026-02-22T22:10:49.642 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3100 IOs, 1335145472 bytes 2026-02-22T22:10:49.686 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3200 IOs, 1372129280 bytes 2026-02-22T22:10:49.727 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3300 IOs, 1395998720 bytes 2026-02-22T22:10:49.766 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3400 IOs, 1415772160 bytes 2026-02-22T22:10:49.801 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3500 IOs, 1429436416 bytes 2026-02-22T22:10:49.849 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3600 IOs, 1479027712 bytes 2026-02-22T22:10:49.888 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3700 IOs, 1496506368 bytes 2026-02-22T22:10:49.951 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3800 IOs, 1583491072 bytes 2026-02-22T22:10:50.004 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3900 IOs, 1640244224 bytes 2026-02-22T22:10:50.046 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4000 IOs, 1665943552 bytes 2026-02-22T22:10:50.088 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4100 IOs, 1693125632 bytes 2026-02-22T22:10:50.140 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4200 IOs, 1749632000 bytes 2026-02-22T22:10:50.195 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4300 IOs, 1810386944 bytes 2026-02-22T22:10:50.239 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4400 IOs, 1848057856 bytes 2026-02-22T22:10:50.309 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4500 IOs, 1946390528 bytes 2026-02-22T22:10:50.362 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4600 IOs, 2003372032 bytes 2026-02-22T22:10:50.425 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4700 IOs, 2084561920 bytes 2026-02-22T22:10:50.480 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4800 IOs, 2150669312 bytes 2026-02-22T22:10:50.527 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4900 IOs, 2194759680 bytes 2026-02-22T22:10:50.562 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5000 IOs, 2209873920 bytes 2026-02-22T22:10:50.649 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5100 IOs, 2322043904 bytes 2026-02-22T22:10:50.687 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5200 IOs, 2341736448 bytes 2026-02-22T22:10:50.730 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5300 IOs, 2364568576 bytes 2026-02-22T22:10:50.777 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5400 IOs, 2413917184 bytes 2026-02-22T22:10:50.821 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5500 IOs, 2442562560 bytes 2026-02-22T22:10:50.853 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5600 IOs, 2457117696 bytes 2026-02-22T22:10:50.929 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5700 IOs, 2568648704 bytes 2026-02-22T22:10:50.991 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5800 IOs, 2652517376 bytes 2026-02-22T22:10:51.056 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5900 IOs, 2732203008 bytes 2026-02-22T22:10:51.115 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6000 IOs, 2802904064 bytes 2026-02-22T22:10:51.166 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6100 IOs, 2859104256 bytes 2026-02-22T22:10:51.209 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6200 IOs, 2901599232 bytes 2026-02-22T22:10:51.256 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6300 IOs, 2943252480 bytes 2026-02-22T22:10:51.287 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6400 IOs, 2943875072 bytes 2026-02-22T22:10:51.318 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6500 IOs, 2944492544 bytes 2026-02-22T22:10:51.374 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6600 IOs, 3018043392 bytes 2026-02-22T22:10:51.420 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6700 IOs, 3056746496 bytes 2026-02-22T22:10:51.456 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6800 IOs, 3072871424 bytes 2026-02-22T22:10:51.500 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6900 IOs, 3116825600 bytes 2026-02-22T22:10:51.535 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7000 IOs, 3127999488 bytes 2026-02-22T22:10:51.581 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7100 IOs, 3169065984 bytes 2026-02-22T22:10:51.643 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7200 IOs, 3220712448 bytes 2026-02-22T22:10:51.693 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7300 IOs, 3267174400 bytes 2026-02-22T22:10:51.744 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7400 IOs, 3325175808 bytes 2026-02-22T22:10:51.794 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7500 IOs, 3376924672 bytes 2026-02-22T22:10:51.834 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7600 IOs, 3407559680 bytes 2026-02-22T22:10:51.889 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7700 IOs, 3462204416 bytes 2026-02-22T22:10:51.945 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7800 IOs, 3517817856 bytes 2026-02-22T22:10:51.986 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7900 IOs, 3552361472 bytes 2026-02-22T22:10:52.033 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8000 IOs, 3600602112 bytes 2026-02-22T22:10:52.105 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8100 IOs, 3710486528 bytes 2026-02-22T22:10:52.156 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8200 IOs, 3759463424 bytes 2026-02-22T22:10:52.185 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8300 IOs, 3760074752 bytes 2026-02-22T22:10:52.227 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8400 IOs, 3796134912 bytes 2026-02-22T22:10:52.257 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8500 IOs, 3796755456 bytes 2026-02-22T22:10:52.291 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8600 IOs, 3806853120 bytes 2026-02-22T22:10:52.320 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8700 IOs, 3808635904 bytes 2026-02-22T22:10:52.362 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8800 IOs, 3847540736 bytes 2026-02-22T22:10:52.408 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8900 IOs, 3884708864 bytes 2026-02-22T22:10:52.471 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9000 IOs, 3967878144 bytes 2026-02-22T22:10:52.517 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9100 IOs, 4004014080 bytes 2026-02-22T22:10:52.572 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9200 IOs, 4065178624 bytes 2026-02-22T22:10:52.622 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9300 IOs, 4093857792 bytes 2026-02-22T22:10:52.671 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9400 IOs, 4147519488 bytes 2026-02-22T22:10:52.719 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9500 IOs, 4173912064 bytes 2026-02-22T22:10:52.767 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9600 IOs, 4223325184 bytes 2026-02-22T22:10:52.816 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9700 IOs, 4273060864 bytes 2026-02-22T22:10:52.853 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9800 IOs, 4294406144 bytes 2026-02-22T22:10:52.904 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9900 IOs, 4349651968 bytes 2026-02-22T22:10:52.961 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10000 IOs, 4413574144 bytes 2026-02-22T22:10:52.997 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10100 IOs, 4432260096 bytes 2026-02-22T22:10:53.037 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10200 IOs, 4452541440 bytes 2026-02-22T22:10:53.085 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10300 IOs, 4510911488 bytes 2026-02-22T22:10:53.114 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10400 IOs, 4511535104 bytes 2026-02-22T22:10:53.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ error_code=124 2026-02-22T22:10:53.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set -e 2026-02-22T22:10:53.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' 124 -eq 124 ']' 2026-02-22T22:10:53.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:10:53.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_present cluster1 mirror image_9 present 2026-02-22T22:10:53.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:10:53.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:10:53.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_9 2026-02-22T22:10:53.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=present 2026-02-22T22:10:53.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:10:53.167 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:10:53.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:10:53.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_9 2026-02-22T22:10:53.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:10:53.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:10:53.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_9 2026-02-22T22:10:53.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_9 2026-02-22T22:10:53.168 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:10:53.195 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image_id=126ff7a396b0 2026-02-22T22:10:53.195 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:10:53.195 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.1 2026-02-22T22:10:53.297 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_present cluster1 mirror image_9 present 126ff7a396b0 2026-02-22T22:10:53.297 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:10:53.297 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:10:53.298 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_9 2026-02-22T22:10:53.298 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=present 2026-02-22T22:10:53.298 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id=126ff7a396b0 2026-02-22T22:10:53.298 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=deleted 2026-02-22T22:10:53.298 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_image_id 2026-02-22T22:10:53.298 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_9 2026-02-22T22:10:53.298 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:10:53.298 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:10:53.298 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_9 2026-02-22T22:10:53.298 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_9 2026-02-22T22:10:53.298 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:10:53.325 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_image_id=126ff7a396b0 2026-02-22T22:10:53.325 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n 126ff7a396b0 2026-02-22T22:10:53.325 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -z 126ff7a396b0 -o 126ff7a396b0 = 126ff7a396b0 2026-02-22T22:10:53.325 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_state=present 2026-02-22T22:10:53.326 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test present = present 2026-02-22T22:10:53.326 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:10:53.326 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ disable_mirror cluster2 mirror image_9 2026-02-22T22:10:53.326 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:10:53.326 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:10:53.326 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_9 2026-02-22T22:10:53.326 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_9 2026-02-22T22:10:54.298 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial026.stderr:2026-02-22T22:10:54.295+0000 7f1e61d4e640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x55fab719e400 scan_remote_mirror_snapshots: failed to locate remote start snapshot: snap_id=193 2026-02-22T22:10:54.298 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial026.stderr:2026-02-22T22:10:54.295+0000 7f1e61d4e640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x55fab719e400 scan_remote_mirror_snapshots: split-brain detected: failed to find matching non-primary snapshot in remote image: local_snap_id_start=193, local_snap_ns=[mirror state=non-primary, complete=1, mirror_peer_uuids=, primary_mirror_uuid=cfa81703-817a-4074-adbc-5e6e200790bf, primary_snap_id=c1, last_copied_object_number=128, snap_seqs={193=18446744073709551614}] 2026-02-22T22:10:59.301 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Mirroring disabled 2026-02-22T22:10:59.306 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_present cluster1 mirror image_9 deleted 2026-02-22T22:10:59.307 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:10:59.307 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:10:59.307 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_9 2026-02-22T22:10:59.307 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=deleted 2026-02-22T22:10:59.307 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:10:59.307 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:10:59.307 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:10:59.307 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_9 2026-02-22T22:10:59.307 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:10:59.308 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:10:59.308 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_9 2026-02-22T22:10:59.308 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_9 2026-02-22T22:10:59.308 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:10:59.331 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_9: (2) No such file or directory 2026-02-22T22:10:59.333 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image_id= 2026-02-22T22:10:59.333 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:10:59.333 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.1 2026-02-22T22:10:59.435 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_present cluster1 mirror image_9 deleted '' 2026-02-22T22:10:59.435 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:10:59.435 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:10:59.435 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_9 2026-02-22T22:10:59.435 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=deleted 2026-02-22T22:10:59.435 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:10:59.435 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=deleted 2026-02-22T22:10:59.436 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_image_id 2026-02-22T22:10:59.436 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_9 2026-02-22T22:10:59.436 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:10:59.436 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:10:59.436 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_9 2026-02-22T22:10:59.436 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_9 2026-02-22T22:10:59.436 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:10:59.457 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_9: (2) No such file or directory 2026-02-22T22:10:59.460 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_image_id= 2026-02-22T22:10:59.461 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:10:59.461 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test deleted = deleted 2026-02-22T22:10:59.461 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:10:59.461 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ purge_snapshots cluster2 mirror image_9 2026-02-22T22:10:59.461 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:10:59.462 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:10:59.462 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_9 2026-02-22T22:10:59.462 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap purge mirror/image_9 2026-02-22T22:10:59.490 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image_retry cluster2 mirror image_9 2026-02-22T22:10:59.490 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:10:59.490 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:10:59.490 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_9 2026-02-22T22:10:59.490 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:10:59.491 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0 2026-02-22T22:10:59.491 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image cluster2 mirror image_9 2026-02-22T22:10:59.492 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:10:59.492 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:10:59.492 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_9 2026-02-22T22:10:59.492 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_9 2026-02-22T22:10:59.519 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 rm mirror/image_9 2026-02-22T22:11:23.180 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2026-02-22T22:11:23.184 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:11:23.185 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for i in `seq 1 10` 2026-02-22T22:11:23.185 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image=image_10 2026-02-22T22:11:23.185 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ create_image cluster2 mirror image_10 512M 2026-02-22T22:11:23.185 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:11:23.185 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T22:11:23.185 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:11:23.185 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T22:11:23.185 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_10 2026-02-22T22:11:23.185 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T22:11:23.185 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local size=128 2026-02-22T22:11:23.185 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -n 512M ']' 2026-02-22T22:11:23.186 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ size=512M 2026-02-22T22:11:23.186 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ shift 2026-02-22T22:11:23.186 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering,exclusive-lock mirror/image_10 2026-02-22T22:11:23.216 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ enable_mirror cluster2 mirror image_10 2026-02-22T22:11:23.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:11:23.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:11:23.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_10 2026-02-22T22:11:23.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local mode=snapshot 2026-02-22T22:11:23.217 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_10 snapshot 2026-02-22T22:11:24.084 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Mirroring enabled 2026-02-22T22:11:24.089 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 info mirror/image_10 2026-02-22T22:11:24.114 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:rbd image 'image_10': 2026-02-22T22:11:24.114 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: size 512 MiB in 128 objects 2026-02-22T22:11:24.114 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: order 22 (4 MiB objects) 2026-02-22T22:11:24.114 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: snapshot_count: 1 2026-02-22T22:11:24.114 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: id: 19baac50edf9 2026-02-22T22:11:24.114 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: block_name_prefix: rbd_data.19baac50edf9 2026-02-22T22:11:24.114 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: format: 2 2026-02-22T22:11:24.114 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: features: layering, exclusive-lock 2026-02-22T22:11:24.114 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: op_features: 2026-02-22T22:11:24.115 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: flags: 2026-02-22T22:11:24.115 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: create_timestamp: Sun Feb 22 22:11:23 2026 2026-02-22T22:11:24.115 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: access_timestamp: Sun Feb 22 22:11:23 2026 2026-02-22T22:11:24.115 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: modify_timestamp: Sun Feb 22 22:11:23 2026 2026-02-22T22:11:24.115 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring state: enabled 2026-02-22T22:11:24.115 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring mode: snapshot 2026-02-22T22:11:24.115 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring global id: bb5f40ba-5902-4758-b557-3987b1986af1 2026-02-22T22:11:24.115 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout: mirroring primary: true 2026-02-22T22:11:24.116 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ stress_write_image cluster2 mirror image_10 2026-02-22T22:11:24.116 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:11:24.116 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:11:24.116 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_10 2026-02-22T22:11:24.116 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-22T22:11:24.117 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local duration=5 2026-02-22T22:11:24.117 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set +e 2026-02-22T22:11:24.117 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ timeout 5s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_10 --debug-rbd=20 --debug-journaler=20 2026-02-22T22:11:24.200 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:100 IOs, 41937920 bytes 2026-02-22T22:11:24.230 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:200 IOs, 49203200 bytes 2026-02-22T22:11:24.268 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:300 IOs, 70918144 bytes 2026-02-22T22:11:24.319 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:400 IOs, 146362368 bytes 2026-02-22T22:11:24.367 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:500 IOs, 175920128 bytes 2026-02-22T22:11:24.396 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:600 IOs, 187615232 bytes 2026-02-22T22:11:24.449 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:700 IOs, 257554432 bytes 2026-02-22T22:11:24.505 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:800 IOs, 311925760 bytes 2026-02-22T22:11:24.553 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:900 IOs, 349451264 bytes 2026-02-22T22:11:24.593 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1000 IOs, 393726976 bytes 2026-02-22T22:11:24.647 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1100 IOs, 440376320 bytes 2026-02-22T22:11:24.689 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1200 IOs, 464005120 bytes 2026-02-22T22:11:24.727 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1300 IOs, 480659456 bytes 2026-02-22T22:11:24.765 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1400 IOs, 508889088 bytes 2026-02-22T22:11:24.816 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1500 IOs, 555225088 bytes 2026-02-22T22:11:24.852 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1600 IOs, 574868480 bytes 2026-02-22T22:11:24.885 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1700 IOs, 588080128 bytes 2026-02-22T22:11:24.927 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1800 IOs, 620187648 bytes 2026-02-22T22:11:24.959 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:1900 IOs, 625854464 bytes 2026-02-22T22:11:25.002 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2000 IOs, 672566272 bytes 2026-02-22T22:11:25.051 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2100 IOs, 726426624 bytes 2026-02-22T22:11:25.081 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2200 IOs, 727044096 bytes 2026-02-22T22:11:25.135 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2300 IOs, 784542720 bytes 2026-02-22T22:11:25.175 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2400 IOs, 808630272 bytes 2026-02-22T22:11:25.223 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2500 IOs, 864108544 bytes 2026-02-22T22:11:25.267 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2600 IOs, 899921920 bytes 2026-02-22T22:11:25.316 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2700 IOs, 944379904 bytes 2026-02-22T22:11:25.350 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2800 IOs, 958594048 bytes 2026-02-22T22:11:25.398 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:2900 IOs, 1009688576 bytes 2026-02-22T22:11:25.452 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3000 IOs, 1071939584 bytes 2026-02-22T22:11:25.506 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3100 IOs, 1129558016 bytes 2026-02-22T22:11:25.561 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3200 IOs, 1188910080 bytes 2026-02-22T22:11:25.606 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3300 IOs, 1240213504 bytes 2026-02-22T22:11:25.650 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3400 IOs, 1285509120 bytes 2026-02-22T22:11:25.715 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3500 IOs, 1337849856 bytes 2026-02-22T22:11:25.752 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3600 IOs, 1354534912 bytes 2026-02-22T22:11:25.798 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3700 IOs, 1394889728 bytes 2026-02-22T22:11:25.845 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3800 IOs, 1438528512 bytes 2026-02-22T22:11:25.911 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:3900 IOs, 1540322304 bytes 2026-02-22T22:11:25.959 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4000 IOs, 1580874752 bytes 2026-02-22T22:11:26.010 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4100 IOs, 1639404544 bytes 2026-02-22T22:11:26.064 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4200 IOs, 1703738368 bytes 2026-02-22T22:11:26.108 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4300 IOs, 1757572096 bytes 2026-02-22T22:11:26.163 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4400 IOs, 1801372672 bytes 2026-02-22T22:11:26.197 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4500 IOs, 1815801856 bytes 2026-02-22T22:11:26.240 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4600 IOs, 1853170688 bytes 2026-02-22T22:11:26.288 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4700 IOs, 1907955712 bytes 2026-02-22T22:11:26.320 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4800 IOs, 1910154240 bytes 2026-02-22T22:11:26.372 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:4900 IOs, 1971015680 bytes 2026-02-22T22:11:26.413 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5000 IOs, 1996687360 bytes 2026-02-22T22:11:26.468 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5100 IOs, 2060573696 bytes 2026-02-22T22:11:26.500 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5200 IOs, 2061180928 bytes 2026-02-22T22:11:26.545 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5300 IOs, 2091546624 bytes 2026-02-22T22:11:26.608 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5400 IOs, 2171534336 bytes 2026-02-22T22:11:26.650 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5500 IOs, 2202779648 bytes 2026-02-22T22:11:26.683 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5600 IOs, 2217765888 bytes 2026-02-22T22:11:26.732 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5700 IOs, 2246962176 bytes 2026-02-22T22:11:26.797 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5800 IOs, 2339324928 bytes 2026-02-22T22:11:26.834 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:5900 IOs, 2371507200 bytes 2026-02-22T22:11:26.883 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6000 IOs, 2403588096 bytes 2026-02-22T22:11:26.934 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6100 IOs, 2455659520 bytes 2026-02-22T22:11:26.996 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6200 IOs, 2538957824 bytes 2026-02-22T22:11:27.046 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6300 IOs, 2591100928 bytes 2026-02-22T22:11:27.105 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6400 IOs, 2661572608 bytes 2026-02-22T22:11:27.140 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6500 IOs, 2671721472 bytes 2026-02-22T22:11:27.204 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6600 IOs, 2745710592 bytes 2026-02-22T22:11:27.240 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6700 IOs, 2761385984 bytes 2026-02-22T22:11:27.304 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6800 IOs, 2853512192 bytes 2026-02-22T22:11:27.337 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:6900 IOs, 2883783680 bytes 2026-02-22T22:11:27.387 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7000 IOs, 2917303296 bytes 2026-02-22T22:11:27.449 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7100 IOs, 3011003392 bytes 2026-02-22T22:11:27.512 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7200 IOs, 3088083968 bytes 2026-02-22T22:11:27.558 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7300 IOs, 3132304384 bytes 2026-02-22T22:11:27.605 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7400 IOs, 3181895680 bytes 2026-02-22T22:11:27.641 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7500 IOs, 3222647808 bytes 2026-02-22T22:11:27.701 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7600 IOs, 3293593600 bytes 2026-02-22T22:11:27.753 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7700 IOs, 3343628288 bytes 2026-02-22T22:11:27.790 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7800 IOs, 3364011008 bytes 2026-02-22T22:11:27.855 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:7900 IOs, 3450316800 bytes 2026-02-22T22:11:27.918 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8000 IOs, 3536533504 bytes 2026-02-22T22:11:27.959 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8100 IOs, 3558353920 bytes 2026-02-22T22:11:27.998 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8200 IOs, 3589171200 bytes 2026-02-22T22:11:28.027 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8300 IOs, 3589786624 bytes 2026-02-22T22:11:28.064 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8400 IOs, 3608928256 bytes 2026-02-22T22:11:28.101 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8500 IOs, 3638860800 bytes 2026-02-22T22:11:28.153 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8600 IOs, 3678627840 bytes 2026-02-22T22:11:28.193 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8700 IOs, 3703567360 bytes 2026-02-22T22:11:28.221 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8800 IOs, 3704180736 bytes 2026-02-22T22:11:28.257 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:8900 IOs, 3723674624 bytes 2026-02-22T22:11:28.312 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9000 IOs, 3785227264 bytes 2026-02-22T22:11:28.364 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9100 IOs, 3843870720 bytes 2026-02-22T22:11:28.403 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9200 IOs, 3869698048 bytes 2026-02-22T22:11:28.449 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9300 IOs, 3911580672 bytes 2026-02-22T22:11:28.485 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9400 IOs, 3926919168 bytes 2026-02-22T22:11:28.528 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9500 IOs, 3970738176 bytes 2026-02-22T22:11:28.599 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9600 IOs, 4073238528 bytes 2026-02-22T22:11:28.636 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9700 IOs, 4086559744 bytes 2026-02-22T22:11:28.678 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9800 IOs, 4121422848 bytes 2026-02-22T22:11:28.715 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:9900 IOs, 4145413120 bytes 2026-02-22T22:11:28.770 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10000 IOs, 4186289152 bytes 2026-02-22T22:11:28.818 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10100 IOs, 4237181952 bytes 2026-02-22T22:11:28.876 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10200 IOs, 4312569856 bytes 2026-02-22T22:11:28.931 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10300 IOs, 4376710144 bytes 2026-02-22T22:11:28.999 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10400 IOs, 4485943296 bytes 2026-02-22T22:11:29.055 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10500 IOs, 4567801856 bytes 2026-02-22T22:11:29.103 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:10600 IOs, 4602123264 bytes 2026-02-22T22:11:29.122 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ error_code=124 2026-02-22T22:11:29.122 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set -e 2026-02-22T22:11:29.122 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' 124 -eq 124 ']' 2026-02-22T22:11:29.122 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:11:29.122 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_present cluster1 mirror image_10 present 2026-02-22T22:11:29.122 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:11:29.122 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:11:29.123 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_10 2026-02-22T22:11:29.123 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=present 2026-02-22T22:11:29.123 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:11:29.123 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:11:29.123 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:11:29.123 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_10 2026-02-22T22:11:29.123 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:11:29.123 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:11:29.123 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_10 2026-02-22T22:11:29.123 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_10 2026-02-22T22:11:29.123 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:11:29.152 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image_id=126f5ebe014f 2026-02-22T22:11:29.152 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:11:29.152 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.1 2026-02-22T22:11:29.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_present cluster1 mirror image_10 present 126f5ebe014f 2026-02-22T22:11:29.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:11:29.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:11:29.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_10 2026-02-22T22:11:29.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=present 2026-02-22T22:11:29.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id=126f5ebe014f 2026-02-22T22:11:29.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=deleted 2026-02-22T22:11:29.253 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_image_id 2026-02-22T22:11:29.254 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_10 2026-02-22T22:11:29.254 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:11:29.254 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:11:29.254 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_10 2026-02-22T22:11:29.254 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_10 2026-02-22T22:11:29.254 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:11:29.283 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_image_id=126f5ebe014f 2026-02-22T22:11:29.283 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n 126f5ebe014f 2026-02-22T22:11:29.283 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -z 126f5ebe014f -o 126f5ebe014f = 126f5ebe014f 2026-02-22T22:11:29.283 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_state=present 2026-02-22T22:11:29.283 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test present = present 2026-02-22T22:11:29.283 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:11:29.283 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ disable_mirror cluster2 mirror image_10 2026-02-22T22:11:29.283 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:11:29.283 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:11:29.283 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_10 2026-02-22T22:11:29.283 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_10 2026-02-22T22:11:30.297 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial026.stderr:2026-02-22T22:11:30.295+0000 7f1e6254f640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x55fab6d95800 scan_remote_mirror_snapshots: failed to locate remote start snapshot: snap_id=194 2026-02-22T22:11:30.297 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial026.stderr:2026-02-22T22:11:30.295+0000 7f1e6254f640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x55fab6d95800 scan_remote_mirror_snapshots: split-brain detected: failed to find matching non-primary snapshot in remote image: local_snap_id_start=194, local_snap_ns=[mirror state=non-primary, complete=1, mirror_peer_uuids=, primary_mirror_uuid=cfa81703-817a-4074-adbc-5e6e200790bf, primary_snap_id=c2, last_copied_object_number=128, snap_seqs={194=18446744073709551614}] 2026-02-22T22:11:35.298 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:Mirroring disabled 2026-02-22T22:11:35.304 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_present cluster1 mirror image_10 deleted 2026-02-22T22:11:35.304 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:11:35.304 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:11:35.304 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_10 2026-02-22T22:11:35.305 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local state=deleted 2026-02-22T22:11:35.305 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:11:35.305 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local s 2026-02-22T22:11:35.305 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:11:35.305 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_10 2026-02-22T22:11:35.305 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:11:35.305 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:11:35.305 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_10 2026-02-22T22:11:35.305 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_10 2026-02-22T22:11:35.305 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:11:35.329 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_10: (2) No such file or directory 2026-02-22T22:11:35.331 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ image_id= 2026-02-22T22:11:35.331 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:11:35.332 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0.1 2026-02-22T22:11:35.433 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test_image_present cluster1 mirror image_10 deleted '' 2026-02-22T22:11:35.433 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:11:35.433 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:11:35.434 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_10 2026-02-22T22:11:35.434 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local test_state=deleted 2026-02-22T22:11:35.434 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image_id= 2026-02-22T22:11:35.434 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_state=deleted 2026-02-22T22:11:35.434 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local current_image_id 2026-02-22T22:11:35.434 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ get_image_id cluster1 mirror image_10 2026-02-22T22:11:35.434 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:11:35.434 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:11:35.434 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local image=image_10 2026-02-22T22:11:35.434 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rbd --cluster=cluster1 info mirror/image_10 2026-02-22T22:11:35.434 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:11:35.455 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:rbd: error opening image image_10: (2) No such file or directory 2026-02-22T22:11:35.457 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ current_image_id= 2026-02-22T22:11:35.457 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n '' 2026-02-22T22:11:35.457 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test deleted = deleted 2026-02-22T22:11:35.457 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:11:35.458 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ purge_snapshots cluster2 mirror image_10 2026-02-22T22:11:35.458 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:11:35.458 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:11:35.458 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_10 2026-02-22T22:11:35.458 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster cluster2 snap purge mirror/image_10 2026-02-22T22:11:35.485 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image_retry cluster2 mirror image_10 2026-02-22T22:11:35.485 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:11:35.485 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:11:35.486 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_10 2026-02-22T22:11:35.486 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:11:35.486 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0 2026-02-22T22:11:35.487 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ remove_image cluster2 mirror image_10 2026-02-22T22:11:35.487 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:11:35.487 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:11:35.487 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local image=image_10 2026-02-22T22:11:35.487 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_10 2026-02-22T22:11:35.514 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rbd --cluster=cluster2 rm mirror/image_10 2026-02-22T22:11:40.118 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.6062951861655336) > weight (0.25) 2026-02-22T22:11:40.118 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.7851600238791927) > weight (0.25) 2026-02-22T22:11:40.118 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.2 2026-02-22T22:11:40.118 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Sent signal 15 2026-02-22T22:11:40.118 INFO:tasks.rbd_mirror_thrash:waiting for 8.713031861743136 secs before reviving daemons 2026-02-22T22:11:40.119 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial026.stderr:2026-02-22T22:11:40.116+0000 7f1e6fd6a640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.2 (PID: 61237) UID: 1000 2026-02-22T22:11:48.832 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.2 2026-02-22T22:11:48.832 INFO:teuthology.orchestra.run:waiting for 600 2026-02-22T22:11:48.832 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.2 2026-02-22T22:11:48.832 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Restarting daemon 2026-02-22T22:11:48.832 DEBUG:teuthology.orchestra.run.trial026:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.2 2026-02-22T22:11:48.834 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Started 2026-02-22T22:11:48.834 INFO:tasks.rbd_mirror_thrash:waiting for 82.41358582193575 secs before thrashing 2026-02-22T22:11:59.551 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2026-02-22T22:11:59.552 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:11:59.553 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ testlog 'TEST: check if removed images'\'' OMAP are removed' 2026-02-22T22:11:59.553 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ tee -a /tmp/tmp.j44LZPowal/rbd-mirror.test.log 2026-02-22T22:11:59.553 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ date '+%F %T' 2026-02-22T22:11:59.555 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ echo 2026-02-22 22:11:59 TEST: check if removed 'images'\''' OMAP are removed 2026-02-22T22:11:59.555 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:2026-02-22 22:11:59 TEST: check if removed images' OMAP are removed 2026-02-22T22:11:59.555 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_in_omap cluster1 mirror 2026-02-22T22:11:59.555 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:11:59.555 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:11:59.555 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_omap_keys cluster1 mirror rbd_mirroring status_global 2026-02-22T22:11:59.555 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:11:59.555 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:11:59.556 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local obj_name=rbd_mirroring 2026-02-22T22:11:59.556 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local filter=status_global 2026-02-22T22:11:59.556 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-02-22T22:11:59.556 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0 2026-02-22T22:11:59.556 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set +e 2026-02-22T22:11:59.557 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ count_omap_keys_with_filter cluster1 mirror rbd_mirroring status_global 2026-02-22T22:11:59.557 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:11:59.558 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:11:59.558 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local obj_name=rbd_mirroring 2026-02-22T22:11:59.558 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local filter=status_global 2026-02-22T22:11:59.558 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ list_omap_keys cluster1 mirror rbd_mirroring 2026-02-22T22:11:59.558 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:11:59.558 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:11:59.558 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local obj_name=rbd_mirroring 2026-02-22T22:11:59.558 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rados --cluster cluster1 -p mirror listomapkeys rbd_mirroring 2026-02-22T22:11:59.558 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ grep -c status_global 2026-02-22T22:11:59.579 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 0 = 0 2026-02-22T22:11:59.580 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ error_code=0 2026-02-22T22:11:59.580 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set -e 2026-02-22T22:11:59.580 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' 0 -eq 0 ']' 2026-02-22T22:11:59.580 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:11:59.580 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_omap_keys cluster1 mirror rbd_mirroring image_ 2026-02-22T22:11:59.580 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:11:59.580 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:11:59.580 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local obj_name=rbd_mirroring 2026-02-22T22:11:59.580 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local filter=image_ 2026-02-22T22:11:59.580 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-02-22T22:11:59.580 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0 2026-02-22T22:11:59.580 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set +e 2026-02-22T22:11:59.580 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ count_omap_keys_with_filter cluster1 mirror rbd_mirroring image_ 2026-02-22T22:11:59.580 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:11:59.580 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:11:59.580 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local obj_name=rbd_mirroring 2026-02-22T22:11:59.580 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local filter=image_ 2026-02-22T22:11:59.581 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ list_omap_keys cluster1 mirror rbd_mirroring 2026-02-22T22:11:59.581 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:11:59.581 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:11:59.581 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local obj_name=rbd_mirroring 2026-02-22T22:11:59.581 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rados --cluster cluster1 -p mirror listomapkeys rbd_mirroring 2026-02-22T22:11:59.581 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ grep -c image_ 2026-02-22T22:11:59.607 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 0 = 0 2026-02-22T22:11:59.607 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ error_code=0 2026-02-22T22:11:59.608 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set -e 2026-02-22T22:11:59.608 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' 0 -eq 0 ']' 2026-02-22T22:11:59.608 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:11:59.608 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_omap_keys cluster1 mirror rbd_mirror_leader image_map 2026-02-22T22:11:59.608 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:11:59.609 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:11:59.609 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local obj_name=rbd_mirror_leader 2026-02-22T22:11:59.609 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local filter=image_map 2026-02-22T22:11:59.609 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-02-22T22:11:59.609 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0 2026-02-22T22:11:59.609 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set +e 2026-02-22T22:11:59.610 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ count_omap_keys_with_filter cluster1 mirror rbd_mirror_leader image_map 2026-02-22T22:11:59.610 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:11:59.610 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:11:59.610 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local obj_name=rbd_mirror_leader 2026-02-22T22:11:59.610 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local filter=image_map 2026-02-22T22:11:59.610 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ list_omap_keys cluster1 mirror rbd_mirror_leader 2026-02-22T22:11:59.611 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster1 2026-02-22T22:11:59.611 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:11:59.611 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local obj_name=rbd_mirror_leader 2026-02-22T22:11:59.611 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rados --cluster cluster1 -p mirror listomapkeys rbd_mirror_leader 2026-02-22T22:11:59.611 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ grep -c image_map 2026-02-22T22:11:59.626 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 0 = 0 2026-02-22T22:11:59.626 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ error_code=0 2026-02-22T22:11:59.626 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set -e 2026-02-22T22:11:59.626 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' 0 -eq 0 ']' 2026-02-22T22:11:59.626 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:11:59.626 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_image_in_omap cluster2 mirror 2026-02-22T22:11:59.626 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:11:59.626 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:11:59.626 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_omap_keys cluster2 mirror rbd_mirroring status_global 2026-02-22T22:11:59.626 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:11:59.626 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:11:59.626 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local obj_name=rbd_mirroring 2026-02-22T22:11:59.626 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local filter=status_global 2026-02-22T22:11:59.626 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-02-22T22:11:59.626 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0 2026-02-22T22:11:59.627 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set +e 2026-02-22T22:11:59.627 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ count_omap_keys_with_filter cluster2 mirror rbd_mirroring status_global 2026-02-22T22:11:59.627 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster2 2026-02-22T22:11:59.627 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:11:59.627 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local obj_name=rbd_mirroring 2026-02-22T22:11:59.627 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local filter=status_global 2026-02-22T22:11:59.627 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ list_omap_keys cluster2 mirror rbd_mirroring 2026-02-22T22:11:59.627 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster2 2026-02-22T22:11:59.627 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:11:59.627 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local obj_name=rbd_mirroring 2026-02-22T22:11:59.627 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rados --cluster cluster2 -p mirror listomapkeys rbd_mirroring 2026-02-22T22:11:59.627 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ grep -c status_global 2026-02-22T22:11:59.648 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 0 = 0 2026-02-22T22:11:59.648 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ error_code=0 2026-02-22T22:11:59.648 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set -e 2026-02-22T22:11:59.648 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' 0 -eq 0 ']' 2026-02-22T22:11:59.648 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:11:59.648 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_omap_keys cluster2 mirror rbd_mirroring image_ 2026-02-22T22:11:59.648 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:11:59.648 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:11:59.648 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local obj_name=rbd_mirroring 2026-02-22T22:11:59.648 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local filter=image_ 2026-02-22T22:11:59.648 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-02-22T22:11:59.648 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0 2026-02-22T22:11:59.649 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set +e 2026-02-22T22:11:59.649 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ count_omap_keys_with_filter cluster2 mirror rbd_mirroring image_ 2026-02-22T22:11:59.649 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster2 2026-02-22T22:11:59.649 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:11:59.649 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local obj_name=rbd_mirroring 2026-02-22T22:11:59.649 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local filter=image_ 2026-02-22T22:11:59.649 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ list_omap_keys cluster2 mirror rbd_mirroring 2026-02-22T22:11:59.649 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster2 2026-02-22T22:11:59.649 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:11:59.649 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local obj_name=rbd_mirroring 2026-02-22T22:11:59.650 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rados --cluster cluster2 -p mirror listomapkeys rbd_mirroring 2026-02-22T22:11:59.650 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ grep -c image_ 2026-02-22T22:11:59.670 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 0 = 0 2026-02-22T22:11:59.670 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ error_code=0 2026-02-22T22:11:59.670 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set -e 2026-02-22T22:11:59.670 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' 0 -eq 0 ']' 2026-02-22T22:11:59.670 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:11:59.670 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ wait_for_omap_keys cluster2 mirror rbd_mirror_leader image_map 2026-02-22T22:11:59.670 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:11:59.670 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local pool=mirror 2026-02-22T22:11:59.670 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local obj_name=rbd_mirror_leader 2026-02-22T22:11:59.670 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local filter=image_map 2026-02-22T22:11:59.670 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-02-22T22:11:59.670 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ sleep 0 2026-02-22T22:11:59.672 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set +e 2026-02-22T22:11:59.672 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ count_omap_keys_with_filter cluster2 mirror rbd_mirror_leader image_map 2026-02-22T22:11:59.672 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster2 2026-02-22T22:11:59.672 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:11:59.672 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local obj_name=rbd_mirror_leader 2026-02-22T22:11:59.672 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local filter=image_map 2026-02-22T22:11:59.672 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ list_omap_keys cluster2 mirror rbd_mirror_leader 2026-02-22T22:11:59.672 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ grep -c image_map 2026-02-22T22:11:59.672 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local cluster=cluster2 2026-02-22T22:11:59.672 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local pool=mirror 2026-02-22T22:11:59.673 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ local obj_name=rbd_mirror_leader 2026-02-22T22:11:59.673 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ rados --cluster cluster2 -p mirror listomapkeys rbd_mirror_leader 2026-02-22T22:11:59.691 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test 0 = 0 2026-02-22T22:11:59.692 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ error_code=0 2026-02-22T22:11:59.692 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set -e 2026-02-22T22:11:59.692 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' 0 -eq 0 ']' 2026-02-22T22:11:59.692 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 0 2026-02-22T22:11:59.692 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ cleanup 0 2026-02-22T22:11:59.692 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local error_code=0 2026-02-22T22:11:59.692 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ set +e 2026-02-22T22:11:59.692 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' 0 -ne 0 ']' 2026-02-22T22:11:59.692 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -z '' ']' 2026-02-22T22:11:59.692 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster instance 2026-02-22T22:11:59.692 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ CEPH_ARGS= 2026-02-22T22:11:59.693 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ ceph --cluster cluster1 osd pool rm mirror mirror --yes-i-really-really-mean-it 2026-02-22T22:12:00.549 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:pool 'mirror' does not exist 2026-02-22T22:12:00.558 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ CEPH_ARGS= 2026-02-22T22:12:00.558 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ ceph --cluster cluster2 osd pool rm mirror mirror --yes-i-really-really-mean-it 2026-02-22T22:12:01.303 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:pool 'mirror' does not exist 2026-02-22T22:12:01.314 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ CEPH_ARGS= 2026-02-22T22:12:01.314 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ ceph --cluster cluster1 osd pool rm mirror_parent mirror_parent --yes-i-really-really-mean-it 2026-02-22T22:12:01.552 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:pool 'mirror_parent' does not exist 2026-02-22T22:12:01.558 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ CEPH_ARGS= 2026-02-22T22:12:01.558 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ ceph --cluster cluster2 osd pool rm mirror_parent mirror_parent --yes-i-really-really-mean-it 2026-02-22T22:12:02.301 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:pool 'mirror_parent' does not exist 2026-02-22T22:12:02.311 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for cluster in "${CLUSTER1}" "${CLUSTER2}" 2026-02-22T22:12:02.311 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ stop_mirrors cluster1 2026-02-22T22:12:02.311 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1 2026-02-22T22:12:02.311 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local sig= 2026-02-22T22:12:02.312 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ seq 0 3 2026-02-22T22:12:02.315 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:12:02.315 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ stop_mirror cluster1:0 '' 2026-02-22T22:12:02.315 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1:0 2026-02-22T22:12:02.315 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local sig= 2026-02-22T22:12:02.315 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n 1 2026-02-22T22:12:02.316 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 2026-02-22T22:12:02.316 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:12:02.316 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ stop_mirror cluster1:1 '' 2026-02-22T22:12:02.316 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1:1 2026-02-22T22:12:02.316 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local sig= 2026-02-22T22:12:02.316 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n 1 2026-02-22T22:12:02.316 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 2026-02-22T22:12:02.316 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:12:02.316 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ stop_mirror cluster1:2 '' 2026-02-22T22:12:02.316 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1:2 2026-02-22T22:12:02.316 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local sig= 2026-02-22T22:12:02.316 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n 1 2026-02-22T22:12:02.316 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 2026-02-22T22:12:02.317 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:12:02.317 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ stop_mirror cluster1:3 '' 2026-02-22T22:12:02.317 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster1:3 2026-02-22T22:12:02.317 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local sig= 2026-02-22T22:12:02.317 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n 1 2026-02-22T22:12:02.317 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 2026-02-22T22:12:02.317 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for cluster in "${CLUSTER1}" "${CLUSTER2}" 2026-02-22T22:12:02.317 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ stop_mirrors cluster2 2026-02-22T22:12:02.317 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2 2026-02-22T22:12:02.317 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local sig= 2026-02-22T22:12:02.317 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:++ seq 0 3 2026-02-22T22:12:02.318 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:12:02.318 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ stop_mirror cluster2:0 '' 2026-02-22T22:12:02.318 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2:0 2026-02-22T22:12:02.318 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local sig= 2026-02-22T22:12:02.318 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n 1 2026-02-22T22:12:02.318 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 2026-02-22T22:12:02.318 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:12:02.318 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ stop_mirror cluster2:1 '' 2026-02-22T22:12:02.318 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2:1 2026-02-22T22:12:02.318 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local sig= 2026-02-22T22:12:02.318 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n 1 2026-02-22T22:12:02.319 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 2026-02-22T22:12:02.319 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:12:02.319 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ stop_mirror cluster2:2 '' 2026-02-22T22:12:02.319 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2:2 2026-02-22T22:12:02.319 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local sig= 2026-02-22T22:12:02.319 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n 1 2026-02-22T22:12:02.319 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 2026-02-22T22:12:02.319 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:12:02.319 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ stop_mirror cluster2:3 '' 2026-02-22T22:12:02.319 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local cluster=cluster2:3 2026-02-22T22:12:02.319 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ local sig= 2026-02-22T22:12:02.319 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test -n 1 2026-02-22T22:12:02.319 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ return 2026-02-22T22:12:02.320 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' -z 1 ']' 2026-02-22T22:12:02.320 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ test '' = /tmp/tmp.j44LZPowal 2026-02-22T22:12:02.320 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ rm -Rf /tmp/tmp.j44LZPowal 2026-02-22T22:12:02.320 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ '[' 0 -eq 0 ']' 2026-02-22T22:12:02.321 INFO:tasks.workunit.cluster1.client.mirror.trial026.stdout:OK 2026-02-22T22:12:02.322 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ echo OK 2026-02-22T22:12:02.322 INFO:tasks.workunit.cluster1.client.mirror.trial026.stderr:+ exit 0 2026-02-22T22:12:02.322 INFO:teuthology.orchestra.run:Running command with timeout 3600 2026-02-22T22:12:02.322 DEBUG:teuthology.orchestra.run.trial026:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror/tmp 2026-02-22T22:12:02.331 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial026.stderr:2026-02-22T22:12:02.328+0000 7efd819cd640 -1 rbd::mirror::LeaderWatcher: 0x55d079874c00 handle_notify_heartbeat: error notifying heartbeat: (2) No such file or directory, releasing leader 2026-02-22T22:12:02.332 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial026.stderr:2026-02-22T22:12:02.328+0000 7efd6e1a6640 -1 rbd::mirror::image_deleter::TrashWatcher: 0x55d079894480 handle_unregister_watcher: error unregistering watcher for trash directory: (2) No such file or directory 2026-02-22T22:12:02.332 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial026.stderr:2026-02-22T22:12:02.328+0000 7efd6e1a6640 -1 rbd::mirror::image_deleter::TrashWatcher: 0x55d0796bc000 handle_unregister_watcher: error unregistering watcher for trash directory: (2) No such file or directory 2026-02-22T22:12:02.332 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial026.stderr:2026-02-22T22:12:02.328+0000 7efd6d9a5640 -1 rbd::mirror::PoolWatcher: 0x55d079d68960 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-02-22T22:12:02.332 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial026.stderr:2026-02-22T22:12:02.328+0000 7efd6e1a6640 -1 rbd::mirror::PoolWatcher: 0x55d07963c5a0 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-02-22T22:12:02.332 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial026.stderr:2026-02-22T22:12:02.328+0000 7efd67999640 -1 rbd::mirror::PoolWatcher: 0x55d07963c780 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-02-22T22:12:02.333 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial026.stderr:2026-02-22T22:12:02.328+0000 7efd67999640 -1 rbd::mirror::PoolWatcher: 0x55d07963cb40 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-02-22T22:12:02.333 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial026.stderr:2026-02-22T22:12:02.328+0000 7efd821ce640 -1 rbd::mirror::LeaderWatcher: 0x55d079874c00 handle_notify_lock_released: error notifying leader lock released: (2) No such file or directory 2026-02-22T22:12:02.333 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial026.stderr:2026-02-22T22:12:02.328+0000 7efd821ce640 -1 rbd::mirror::LeaderWatcher: 0x55d079874c00 handle_release_leader_lock: error releasing lock: (2) No such file or directory 2026-02-22T22:12:02.357 INFO:tasks.workunit:Stopping ['rbd/rbd_mirror_stress.sh'] on cluster1.client.mirror... 2026-02-22T22:12:02.358 DEBUG:teuthology.orchestra.run.trial026:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.cluster1.client.mirror /home/ubuntu/cephtest/clone.cluster1.client.mirror 2026-02-22T22:12:02.387 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial026.stderr:2026-02-22T22:12:02.384+0000 7efd821ce640 -1 rbd::mirror::LeaderWatcher: 0x55d07a434300 handle_notify_heartbeat: error notifying heartbeat: (2) No such file or directory, releasing leader 2026-02-22T22:12:02.388 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial026.stderr:2026-02-22T22:12:02.384+0000 7efd7b1c0640 -1 rbd::mirror::image_deleter::TrashWatcher: 0x55d079ac5440 handle_unregister_watcher: error unregistering watcher for trash directory: (2) No such file or directory 2026-02-22T22:12:02.388 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial026.stderr:2026-02-22T22:12:02.384+0000 7efd7b9c1640 -1 rbd::mirror::image_deleter::TrashWatcher: 0x55d0796bcb40 handle_unregister_watcher: error unregistering watcher for trash directory: (2) No such file or directory 2026-02-22T22:12:02.388 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial026.stderr:2026-02-22T22:12:02.384+0000 7efd7b9c1640 -1 rbd::mirror::image_deleter::TrashWatcher: 0x55d079ac4900 handle_unregister_watcher: error unregistering watcher for trash directory: (2) No such file or directory 2026-02-22T22:12:02.388 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial026.stderr:2026-02-22T22:12:02.384+0000 7efd7b1c0640 -1 rbd::mirror::PoolWatcher: 0x55d079de81e0 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-02-22T22:12:02.388 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial026.stderr:2026-02-22T22:12:02.384+0000 7efd7b1c0640 -1 rbd::mirror::PoolWatcher: 0x55d079de8000 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-02-22T22:12:02.388 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial026.stderr:2026-02-22T22:12:02.384+0000 7efd7b9c1640 -1 rbd::mirror::PoolWatcher: 0x55d079de83c0 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-02-22T22:12:02.388 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial026.stderr:2026-02-22T22:12:02.385+0000 7efd751b4640 -1 rbd::mirror::PoolWatcher: 0x55d079de8960 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-02-22T22:12:02.388 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial026.stderr:2026-02-22T22:12:02.385+0000 7efd751b4640 -1 rbd::mirror::PoolWatcher: 0x55d079de85a0 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-02-22T22:12:02.388 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial026.stderr:2026-02-22T22:12:02.385+0000 7efd751b4640 -1 rbd::mirror::PoolWatcher: 0x55d079de8780 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-02-22T22:12:02.388 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial026.stderr:2026-02-22T22:12:02.385+0000 7efd819cd640 -1 rbd::mirror::LeaderWatcher: 0x55d07a434300 handle_notify_lock_released: error notifying leader lock released: (2) No such file or directory 2026-02-22T22:12:02.388 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial026.stderr:2026-02-22T22:12:02.385+0000 7efd819cd640 -1 rbd::mirror::LeaderWatcher: 0x55d07a434300 handle_release_leader_lock: error releasing lock: (2) No such file or directory 2026-02-22T22:12:02.686 DEBUG:teuthology.parallel:result is None 2026-02-22T22:12:02.686 DEBUG:teuthology.orchestra.run.trial026:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror 2026-02-22T22:12:02.711 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror 2026-02-22T22:12:02.711 DEBUG:teuthology.orchestra.run.trial026:> rmdir -- /home/ubuntu/cephtest/mnt.cluster1.mirror 2026-02-22T22:12:02.767 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror 2026-02-22T22:12:02.767 DEBUG:teuthology.run_tasks:Unwinding manager rbd-mirror-thrash 2026-02-22T22:12:02.778 INFO:tasks.rbd_mirror_thrash:joining rbd_mirror_thrash 2026-02-22T22:12:02.778 INFO:tasks.rbd_mirror_thrash:stat['kill'] = 18 2026-02-22T22:12:02.779 INFO:tasks.rbd_mirror_thrash:done joining 2026-02-22T22:12:02.779 DEBUG:teuthology.run_tasks:Unwinding manager rbd-mirror-thrash 2026-02-22T22:12:02.799 INFO:tasks.rbd_mirror_thrash:joining rbd_mirror_thrash 2026-02-22T22:12:02.799 INFO:tasks.rbd_mirror_thrash:stat['kill'] = 16 2026-02-22T22:12:02.799 INFO:tasks.rbd_mirror_thrash:done joining 2026-02-22T22:12:02.799 DEBUG:teuthology.run_tasks:Unwinding manager rbd-mirror 2026-02-22T22:12:02.818 DEBUG:tasks.rbd_mirror.cluster2.client.mirror.3:waiting for process to exit 2026-02-22T22:12:02.818 INFO:teuthology.orchestra.run:waiting for 300 2026-02-22T22:12:02.821 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial026.stderr:2026-02-22T22:12:02.817+0000 7f39c9549640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.3 (PID: 82810) UID: 1000 2026-02-22T22:12:02.821 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial026.stderr:2026-02-22T22:12:02.817+0000 7f39c8d48640 -1 rbd::mirror::InstanceWatcher: 0x562b30662700 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.6483 object: (2) No such file or directory 2026-02-22T22:12:02.822 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial026.stderr:2026-02-22T22:12:02.817+0000 7f39c8547640 -1 rbd::mirror::InstanceWatcher: 0x562b30a11880 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.6481 object: (2) No such file or directory 2026-02-22T22:12:02.822 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial026.stderr:2026-02-22T22:12:02.817+0000 7f39c8547640 -1 rbd::mirror::InstanceWatcher: 0x562b30a11c00 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.6481 object: (2) No such file or directory 2026-02-22T22:12:02.822 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial026.stderr:2026-02-22T22:12:02.817+0000 7f39b4d20640 -1 rbd::mirror::InstanceWatcher: 0x562b30662700 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-22T22:12:02.822 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial026.stderr:2026-02-22T22:12:02.817+0000 7f39c253b640 -1 rbd::mirror::InstanceWatcher: 0x562b30a11880 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-22T22:12:02.822 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial026.stderr:2026-02-22T22:12:02.817+0000 7f39c253b640 -1 rbd::mirror::InstanceWatcher: 0x562b30a11c00 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-22T22:12:02.822 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial026.stderr:2026-02-22T22:12:02.818+0000 7f39ae513640 -1 rbd::mirror::MirrorStatusUpdater 0x562b30456b40 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-22T22:12:02.822 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial026.stderr:2026-02-22T22:12:02.818+0000 7f39c8547640 -1 rbd::mirror::NamespaceReplayer: 0x562b30710000 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-22T22:12:02.822 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial026.stderr:2026-02-22T22:12:02.818+0000 7f39bbd2e640 -1 rbd::mirror::MirrorStatusUpdater 0x562b302805a0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-22T22:12:02.822 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial026.stderr:2026-02-22T22:12:02.818+0000 7f39bbd2e640 -1 rbd::mirror::MirrorStatusUpdater 0x562b30280360 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-22T22:12:02.822 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial026.stderr:2026-02-22T22:12:02.818+0000 7f39c8d48640 -1 rbd::mirror::NamespaceReplayer: 0x562b3027e000 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-22T22:12:02.823 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial026.stderr:2026-02-22T22:12:02.818+0000 7f39c8d48640 -1 rbd::mirror::NamespaceReplayer: 0x562b3027e1a0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-22T22:12:02.823 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial026.stderr:2026-02-22T22:12:02.818+0000 7f39b451f640 -1 rbd::mirror::MirrorStatusUpdater 0x562b30456a20 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-22T22:12:02.823 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial026.stderr:2026-02-22T22:12:02.818+0000 7f39c1d3a640 -1 rbd::mirror::MirrorStatusUpdater 0x562b30280000 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-22T22:12:02.823 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial026.stderr:2026-02-22T22:12:02.818+0000 7f39c8547640 -1 rbd::mirror::NamespaceReplayer: 0x562b30710000 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-22T22:12:02.823 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial026.stderr:2026-02-22T22:12:02.818+0000 7f39c1d3a640 -1 rbd::mirror::MirrorStatusUpdater 0x562b30280120 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-22T22:12:02.823 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial026.stderr:2026-02-22T22:12:02.818+0000 7f39c8547640 -1 rbd::mirror::NamespaceReplayer: 0x562b3027e000 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-22T22:12:02.823 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial026.stderr:2026-02-22T22:12:02.818+0000 7f39c8d48640 -1 rbd::mirror::NamespaceReplayer: 0x562b3027e1a0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-22T22:12:02.823 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial026.stderr:2026-02-22T22:12:02.818+0000 7f39c8d48640 -1 rbd::mirror::LeaderWatcher: 0x562b30632300 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2026-02-22T22:12:02.823 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial026.stderr:2026-02-22T22:12:02.818+0000 7f39c8d48640 -1 rbd::mirror::InstanceWatcher: 0x562b30662380 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.6483 object: (2) No such file or directory 2026-02-22T22:12:02.823 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial026.stderr:2026-02-22T22:12:02.818+0000 7f39b4d20640 -1 rbd::mirror::InstanceWatcher: 0x562b30662380 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-22T22:12:02.823 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial026.stderr:2026-02-22T22:12:02.818+0000 7f39add12640 -1 rbd::mirror::MirrorStatusUpdater 0x562b302806c0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-22T22:12:02.823 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial026.stderr:2026-02-22T22:12:02.818+0000 7f39c8d48640 -1 rbd::mirror::NamespaceReplayer: 0x562b3027e4e0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-22T22:12:02.824 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial026.stderr:2026-02-22T22:12:02.818+0000 7f39b451f640 -1 rbd::mirror::MirrorStatusUpdater 0x562b3043bc20 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-22T22:12:02.824 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial026.stderr:2026-02-22T22:12:02.818+0000 7f39c8547640 -1 rbd::mirror::NamespaceReplayer: 0x562b3027e4e0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-22T22:12:02.825 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial026.stderr:2026-02-22T22:12:02.820+0000 7f39c8d48640 -1 rbd::mirror::LeaderWatcher: 0x562b2ffb9b00 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2026-02-22T22:12:02.825 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial026.stderr:2026-02-22T22:12:02.820+0000 7f39c8547640 -1 rbd::mirror::InstanceWatcher: 0x562b30a11500 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.6481 object: (2) No such file or directory 2026-02-22T22:12:02.825 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial026.stderr:2026-02-22T22:12:02.820+0000 7f39c253b640 -1 rbd::mirror::InstanceWatcher: 0x562b30a11500 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-22T22:12:02.825 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial026.stderr:2026-02-22T22:12:02.820+0000 7f39bb52d640 -1 rbd::mirror::MirrorStatusUpdater 0x562b2ffa46c0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-22T22:12:02.825 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial026.stderr:2026-02-22T22:12:02.820+0000 7f39c8d48640 -1 rbd::mirror::NamespaceReplayer: 0x562b2fe2a1a0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-22T22:12:02.825 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial026.stderr:2026-02-22T22:12:02.820+0000 7f39c1d3a640 -1 rbd::mirror::MirrorStatusUpdater 0x562b2ff33560 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-22T22:12:02.825 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial026.stderr:2026-02-22T22:12:02.820+0000 7f39c8547640 -1 rbd::mirror::NamespaceReplayer: 0x562b2fe2a1a0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-22T22:12:02.831 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Stopped 2026-02-22T22:12:02.832 DEBUG:teuthology.run_tasks:Unwinding manager rbd-mirror 2026-02-22T22:12:02.854 DEBUG:tasks.rbd_mirror.cluster2.client.mirror.2:waiting for process to exit 2026-02-22T22:12:02.854 INFO:teuthology.orchestra.run:waiting for 300 2026-02-22T22:12:02.873 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial026.stderr:2026-02-22T22:12:02.871+0000 7f5f9fe8f640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2 (PID: 71448) UID: 1000 2026-02-22T22:12:02.874 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial026.stderr:2026-02-22T22:12:02.871+0000 7f5f9ee8d640 -1 rbd::mirror::InstanceWatcher: 0x5639529f6380 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5798 object: (2) No such file or directory 2026-02-22T22:12:02.875 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial026.stderr:2026-02-22T22:12:02.871+0000 7f5f9f68e640 -1 rbd::mirror::InstanceWatcher: 0x56395309f880 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5795 object: (2) No such file or directory 2026-02-22T22:12:02.875 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial026.stderr:2026-02-22T22:12:02.871+0000 7f5f9f68e640 -1 rbd::mirror::InstanceWatcher: 0x563952646000 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5795 object: (2) No such file or directory 2026-02-22T22:12:02.875 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial026.stderr:2026-02-22T22:12:02.871+0000 7f5f8b666640 -1 rbd::mirror::InstanceWatcher: 0x5639529f6380 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-22T22:12:02.875 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial026.stderr:2026-02-22T22:12:02.871+0000 7f5f98e81640 -1 rbd::mirror::InstanceWatcher: 0x56395309f880 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-22T22:12:02.875 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial026.stderr:2026-02-22T22:12:02.871+0000 7f5f98680640 -1 rbd::mirror::InstanceWatcher: 0x563952646000 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-22T22:12:02.875 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial026.stderr:2026-02-22T22:12:02.872+0000 7f5f84e59640 -1 rbd::mirror::MirrorStatusUpdater 0x563952814c60 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-22T22:12:02.875 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial026.stderr:2026-02-22T22:12:02.872+0000 7f5f9ee8d640 -1 rbd::mirror::NamespaceReplayer: 0x563952aae000 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-22T22:12:02.875 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial026.stderr:2026-02-22T22:12:02.872+0000 7f5f8ae65640 -1 rbd::mirror::MirrorStatusUpdater 0x5639528150e0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-22T22:12:02.875 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial026.stderr:2026-02-22T22:12:02.872+0000 7f5f9f68e640 -1 rbd::mirror::NamespaceReplayer: 0x563952aae000 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-22T22:12:02.875 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial026.stderr:2026-02-22T22:12:02.872+0000 7f5f9f68e640 -1 rbd::mirror::LeaderWatcher: 0x5639529ce300 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2026-02-22T22:12:02.875 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial026.stderr:2026-02-22T22:12:02.872+0000 7f5f9f68e640 -1 rbd::mirror::InstanceWatcher: 0x56395309fc00 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5798 object: (2) No such file or directory 2026-02-22T22:12:02.875 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial026.stderr:2026-02-22T22:12:02.872+0000 7f5f8b666640 -1 rbd::mirror::InstanceWatcher: 0x56395309fc00 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-22T22:12:02.876 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial026.stderr:2026-02-22T22:12:02.872+0000 7f5f84658640 -1 rbd::mirror::MirrorStatusUpdater 0x5639528147e0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-22T22:12:02.876 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial026.stderr:2026-02-22T22:12:02.872+0000 7f5f9f68e640 -1 rbd::mirror::NamespaceReplayer: 0x56395260e4e0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-22T22:12:02.876 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial026.stderr:2026-02-22T22:12:02.872+0000 7f5f8ae65640 -1 rbd::mirror::MirrorStatusUpdater 0x5639527c78c0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-22T22:12:02.876 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial026.stderr:2026-02-22T22:12:02.872+0000 7f5f9ee8d640 -1 rbd::mirror::NamespaceReplayer: 0x56395260e4e0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-22T22:12:02.877 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial026.stderr:2026-02-22T22:12:02.874+0000 7f5f91e73640 -1 rbd::mirror::MirrorStatusUpdater 0x5639526105a0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-22T22:12:02.877 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial026.stderr:2026-02-22T22:12:02.874+0000 7f5f91e73640 -1 rbd::mirror::MirrorStatusUpdater 0x563952610360 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-22T22:12:02.877 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial026.stderr:2026-02-22T22:12:02.874+0000 7f5f9f68e640 -1 rbd::mirror::NamespaceReplayer: 0x56395260e000 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-22T22:12:02.877 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial026.stderr:2026-02-22T22:12:02.874+0000 7f5f9f68e640 -1 rbd::mirror::NamespaceReplayer: 0x56395260e1a0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-22T22:12:02.877 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial026.stderr:2026-02-22T22:12:02.874+0000 7f5f98e81640 -1 rbd::mirror::MirrorStatusUpdater 0x563952610000 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-22T22:12:02.877 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial026.stderr:2026-02-22T22:12:02.874+0000 7f5f98e81640 -1 rbd::mirror::MirrorStatusUpdater 0x563952610120 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-22T22:12:02.878 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial026.stderr:2026-02-22T22:12:02.874+0000 7f5f9ee8d640 -1 rbd::mirror::NamespaceReplayer: 0x56395260e000 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-22T22:12:02.878 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial026.stderr:2026-02-22T22:12:02.874+0000 7f5f9f68e640 -1 rbd::mirror::NamespaceReplayer: 0x56395260e1a0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-22T22:12:02.878 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial026.stderr:2026-02-22T22:12:02.874+0000 7f5f9ee8d640 -1 rbd::mirror::LeaderWatcher: 0x563953241b00 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2026-02-22T22:12:02.878 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial026.stderr:2026-02-22T22:12:02.874+0000 7f5f9f68e640 -1 rbd::mirror::InstanceWatcher: 0x56395309f500 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5795 object: (2) No such file or directory 2026-02-22T22:12:02.878 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial026.stderr:2026-02-22T22:12:02.874+0000 7f5f98680640 -1 rbd::mirror::InstanceWatcher: 0x56395309f500 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-22T22:12:02.878 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial026.stderr:2026-02-22T22:12:02.874+0000 7f5f92674640 -1 rbd::mirror::MirrorStatusUpdater 0x5639532346c0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-22T22:12:02.878 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial026.stderr:2026-02-22T22:12:02.874+0000 7f5f9f68e640 -1 rbd::mirror::NamespaceReplayer: 0x5639524a21a0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-22T22:12:02.878 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial026.stderr:2026-02-22T22:12:02.875+0000 7f5f98e81640 -1 rbd::mirror::MirrorStatusUpdater 0x5639531cb560 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-22T22:12:02.878 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial026.stderr:2026-02-22T22:12:02.875+0000 7f5f9ee8d640 -1 rbd::mirror::NamespaceReplayer: 0x5639524a21a0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-22T22:12:02.886 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Stopped 2026-02-22T22:12:02.886 DEBUG:teuthology.run_tasks:Unwinding manager rbd-mirror 2026-02-22T22:12:02.902 DEBUG:tasks.rbd_mirror.cluster2.client.mirror.1:waiting for process to exit 2026-02-22T22:12:02.902 INFO:teuthology.orchestra.run:waiting for 300 2026-02-22T22:12:02.926 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial026.stderr:2026-02-22T22:12:02.925+0000 7efd829cf640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 (PID: 74787) UID: 1000 2026-02-22T22:12:02.927 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial026.stderr:2026-02-22T22:12:02.925+0000 7efd819cd640 -1 rbd::mirror::InstanceWatcher: 0x55d07a19f880 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5984 object: (2) No such file or directory 2026-02-22T22:12:02.927 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial026.stderr:2026-02-22T22:12:02.925+0000 7efd819cd640 -1 rbd::mirror::InstanceWatcher: 0x55d079738000 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5984 object: (2) No such file or directory 2026-02-22T22:12:02.927 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial026.stderr:2026-02-22T22:12:02.925+0000 7efd819cd640 -1 rbd::mirror::InstanceWatcher: 0x55d07a19fc00 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5990 object: (2) No such file or directory 2026-02-22T22:12:02.927 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial026.stderr:2026-02-22T22:12:02.925+0000 7efd7b1c0640 -1 rbd::mirror::InstanceWatcher: 0x55d07a19f880 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-22T22:12:02.927 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial026.stderr:2026-02-22T22:12:02.925+0000 7efd7b9c1640 -1 rbd::mirror::InstanceWatcher: 0x55d079738000 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-22T22:12:02.927 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial026.stderr:2026-02-22T22:12:02.925+0000 7efd6e1a6640 -1 rbd::mirror::InstanceWatcher: 0x55d07a19fc00 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-22T22:12:02.928 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial026.stderr:2026-02-22T22:12:02.926+0000 7efd749b3640 -1 rbd::mirror::MirrorStatusUpdater 0x55d0797085a0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-22T22:12:02.928 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial026.stderr:2026-02-22T22:12:02.926+0000 7efd749b3640 -1 rbd::mirror::MirrorStatusUpdater 0x55d079708360 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-22T22:12:02.929 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial026.stderr:2026-02-22T22:12:02.926+0000 7efd67198640 -1 rbd::mirror::MirrorStatusUpdater 0x55d0798f4c60 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-22T22:12:02.929 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial026.stderr:2026-02-22T22:12:02.926+0000 7efd821ce640 -1 rbd::mirror::NamespaceReplayer: 0x55d079706000 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-22T22:12:02.929 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial026.stderr:2026-02-22T22:12:02.926+0000 7efd821ce640 -1 rbd::mirror::NamespaceReplayer: 0x55d0797061a0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-22T22:12:02.929 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial026.stderr:2026-02-22T22:12:02.926+0000 7efd821ce640 -1 rbd::mirror::NamespaceReplayer: 0x55d079b9e000 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-22T22:12:02.929 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial026.stderr:2026-02-22T22:12:02.926+0000 7efd7b9c1640 -1 rbd::mirror::MirrorStatusUpdater 0x55d079708000 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-22T22:12:02.929 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial026.stderr:2026-02-22T22:12:02.926+0000 7efd7b9c1640 -1 rbd::mirror::MirrorStatusUpdater 0x55d079708120 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-22T22:12:02.929 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial026.stderr:2026-02-22T22:12:02.926+0000 7efd819cd640 -1 rbd::mirror::NamespaceReplayer: 0x55d079706000 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-22T22:12:02.929 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial026.stderr:2026-02-22T22:12:02.926+0000 7efd819cd640 -1 rbd::mirror::NamespaceReplayer: 0x55d0797061a0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-22T22:12:02.929 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial026.stderr:2026-02-22T22:12:02.926+0000 7efd6d9a5640 -1 rbd::mirror::MirrorStatusUpdater 0x55d0798f50e0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-22T22:12:02.929 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial026.stderr:2026-02-22T22:12:02.926+0000 7efd819cd640 -1 rbd::mirror::NamespaceReplayer: 0x55d079b9e000 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-22T22:12:02.929 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial026.stderr:2026-02-22T22:12:02.926+0000 7efd821ce640 -1 rbd::mirror::LeaderWatcher: 0x55d079874c00 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2026-02-22T22:12:02.929 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial026.stderr:2026-02-22T22:12:02.926+0000 7efd821ce640 -1 rbd::mirror::InstanceWatcher: 0x55d079ae6380 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5990 object: (2) No such file or directory 2026-02-22T22:12:02.930 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial026.stderr:2026-02-22T22:12:02.926+0000 7efd6e1a6640 -1 rbd::mirror::InstanceWatcher: 0x55d079ae6380 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-22T22:12:02.930 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial026.stderr:2026-02-22T22:12:02.926+0000 7efd67999640 -1 rbd::mirror::MirrorStatusUpdater 0x55d0798f47e0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-22T22:12:02.930 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial026.stderr:2026-02-22T22:12:02.926+0000 7efd819cd640 -1 rbd::mirror::NamespaceReplayer: 0x55d0797064e0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-22T22:12:02.930 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial026.stderr:2026-02-22T22:12:02.926+0000 7efd6d9a5640 -1 rbd::mirror::MirrorStatusUpdater 0x55d0798c98c0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-22T22:12:02.930 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial026.stderr:2026-02-22T22:12:02.926+0000 7efd821ce640 -1 rbd::mirror::NamespaceReplayer: 0x55d0797064e0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-22T22:12:02.930 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial026.stderr:2026-02-22T22:12:02.928+0000 7efd819cd640 -1 rbd::mirror::LeaderWatcher: 0x55d07a434300 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2026-02-22T22:12:02.931 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial026.stderr:2026-02-22T22:12:02.928+0000 7efd819cd640 -1 rbd::mirror::InstanceWatcher: 0x55d07a19f500 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5984 object: (2) No such file or directory 2026-02-22T22:12:02.931 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial026.stderr:2026-02-22T22:12:02.928+0000 7efd7b1c0640 -1 rbd::mirror::InstanceWatcher: 0x55d07a19f500 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-22T22:12:02.931 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial026.stderr:2026-02-22T22:12:02.928+0000 7efd751b4640 -1 rbd::mirror::MirrorStatusUpdater 0x55d07a3326c0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-22T22:12:02.931 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial026.stderr:2026-02-22T22:12:02.928+0000 7efd819cd640 -1 rbd::mirror::NamespaceReplayer: 0x55d0796101a0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-22T22:12:02.931 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial026.stderr:2026-02-22T22:12:02.928+0000 7efd7b9c1640 -1 rbd::mirror::MirrorStatusUpdater 0x55d07a2d1560 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-22T22:12:02.931 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial026.stderr:2026-02-22T22:12:02.928+0000 7efd821ce640 -1 rbd::mirror::NamespaceReplayer: 0x55d0796101a0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-22T22:12:02.940 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Stopped 2026-02-22T22:12:02.940 DEBUG:teuthology.run_tasks:Unwinding manager rbd-mirror 2026-02-22T22:12:02.954 DEBUG:tasks.rbd_mirror.cluster2.client.mirror.0:waiting for process to exit 2026-02-22T22:12:02.954 INFO:teuthology.orchestra.run:waiting for 300 2026-02-22T22:12:02.980 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial026.stderr:2026-02-22T22:12:02.979+0000 7f505979f640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 (PID: 83813) UID: 1000 2026-02-22T22:12:02.981 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial026.stderr:2026-02-22T22:12:02.979+0000 7f5058f9e640 -1 rbd::mirror::InstanceWatcher: 0x5603b77b9880 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.6547 object: (2) No such file or directory 2026-02-22T22:12:02.981 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial026.stderr:2026-02-22T22:12:02.979+0000 7f5058f9e640 -1 rbd::mirror::InstanceWatcher: 0x5603b77b9c00 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.6547 object: (2) No such file or directory 2026-02-22T22:12:02.981 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial026.stderr:2026-02-22T22:12:02.979+0000 7f5058f9e640 -1 rbd::mirror::InstanceWatcher: 0x5603b7400e00 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.6550 object: (2) No such file or directory 2026-02-22T22:12:02.982 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial026.stderr:2026-02-22T22:12:02.979+0000 7f5052791640 -1 rbd::mirror::InstanceWatcher: 0x5603b77b9880 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-22T22:12:02.982 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial026.stderr:2026-02-22T22:12:02.979+0000 7f5052791640 -1 rbd::mirror::InstanceWatcher: 0x5603b77b9c00 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-22T22:12:02.982 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial026.stderr:2026-02-22T22:12:02.979+0000 7f5044f76640 -1 rbd::mirror::InstanceWatcher: 0x5603b7400e00 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-22T22:12:02.982 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial026.stderr:2026-02-22T22:12:02.980+0000 7f504b783640 -1 rbd::mirror::MirrorStatusUpdater 0x5603b7020360 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-22T22:12:02.982 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial026.stderr:2026-02-22T22:12:02.980+0000 7f504b783640 -1 rbd::mirror::MirrorStatusUpdater 0x5603b70205a0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-22T22:12:02.982 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial026.stderr:2026-02-22T22:12:02.980+0000 7f505879d640 -1 rbd::mirror::NamespaceReplayer: 0x5603b701e000 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-22T22:12:02.982 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial026.stderr:2026-02-22T22:12:02.980+0000 7f503e769640 -1 rbd::mirror::MirrorStatusUpdater 0x5603b7020c60 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-22T22:12:02.982 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial026.stderr:2026-02-22T22:12:02.980+0000 7f505879d640 -1 rbd::mirror::NamespaceReplayer: 0x5603b701e1a0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-22T22:12:02.982 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial026.stderr:2026-02-22T22:12:02.980+0000 7f505879d640 -1 rbd::mirror::NamespaceReplayer: 0x5603b74b4000 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-22T22:12:02.982 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial026.stderr:2026-02-22T22:12:02.980+0000 7f5051f90640 -1 rbd::mirror::MirrorStatusUpdater 0x5603b7020000 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-22T22:12:02.982 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial026.stderr:2026-02-22T22:12:02.980+0000 7f5051f90640 -1 rbd::mirror::MirrorStatusUpdater 0x5603b7020120 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-22T22:12:02.982 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial026.stderr:2026-02-22T22:12:02.980+0000 7f5058f9e640 -1 rbd::mirror::NamespaceReplayer: 0x5603b701e000 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-22T22:12:02.982 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial026.stderr:2026-02-22T22:12:02.980+0000 7f5058f9e640 -1 rbd::mirror::NamespaceReplayer: 0x5603b701e1a0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-22T22:12:02.982 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial026.stderr:2026-02-22T22:12:02.980+0000 7f5044775640 -1 rbd::mirror::MirrorStatusUpdater 0x5603b7214a20 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-22T22:12:02.982 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial026.stderr:2026-02-22T22:12:02.980+0000 7f505879d640 -1 rbd::mirror::NamespaceReplayer: 0x5603b74b4000 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-22T22:12:02.982 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial026.stderr:2026-02-22T22:12:02.980+0000 7f505879d640 -1 rbd::mirror::LeaderWatcher: 0x5603b73cc300 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2026-02-22T22:12:02.982 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial026.stderr:2026-02-22T22:12:02.980+0000 7f505879d640 -1 rbd::mirror::InstanceWatcher: 0x5603b7400380 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.6550 object: (2) No such file or directory 2026-02-22T22:12:02.983 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial026.stderr:2026-02-22T22:12:02.980+0000 7f5044f76640 -1 rbd::mirror::InstanceWatcher: 0x5603b7400380 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-22T22:12:02.983 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial026.stderr:2026-02-22T22:12:02.981+0000 7f503e769640 -1 rbd::mirror::MirrorStatusUpdater 0x5603b70206c0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-22T22:12:02.983 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial026.stderr:2026-02-22T22:12:02.981+0000 7f505879d640 -1 rbd::mirror::NamespaceReplayer: 0x5603b701e4e0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-22T22:12:02.983 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial026.stderr:2026-02-22T22:12:02.981+0000 7f5044775640 -1 rbd::mirror::MirrorStatusUpdater 0x5603b71ebc20 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-22T22:12:02.983 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial026.stderr:2026-02-22T22:12:02.981+0000 7f5058f9e640 -1 rbd::mirror::NamespaceReplayer: 0x5603b701e4e0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-22T22:12:02.984 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial026.stderr:2026-02-22T22:12:02.982+0000 7f5058f9e640 -1 rbd::mirror::LeaderWatcher: 0x5603b6d57b00 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2026-02-22T22:12:02.984 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial026.stderr:2026-02-22T22:12:02.982+0000 7f5058f9e640 -1 rbd::mirror::InstanceWatcher: 0x5603b77b9500 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.6547 object: (2) No such file or directory 2026-02-22T22:12:02.984 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial026.stderr:2026-02-22T22:12:02.982+0000 7f5052791640 -1 rbd::mirror::InstanceWatcher: 0x5603b77b9500 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-22T22:12:02.984 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial026.stderr:2026-02-22T22:12:02.982+0000 7f504bf84640 -1 rbd::mirror::MirrorStatusUpdater 0x5603b6d446c0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-22T22:12:02.984 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial026.stderr:2026-02-22T22:12:02.982+0000 7f505879d640 -1 rbd::mirror::NamespaceReplayer: 0x5603b6c441a0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-22T22:12:02.984 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial026.stderr:2026-02-22T22:12:02.983+0000 7f5051f90640 -1 rbd::mirror::MirrorStatusUpdater 0x5603b6cd7560 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-22T22:12:02.984 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial026.stderr:2026-02-22T22:12:02.983+0000 7f5058f9e640 -1 rbd::mirror::NamespaceReplayer: 0x5603b6c441a0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-22T22:12:02.993 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Stopped 2026-02-22T22:12:02.993 DEBUG:teuthology.run_tasks:Unwinding manager rbd-mirror 2026-02-22T22:12:03.008 DEBUG:tasks.rbd_mirror.cluster1.client.mirror.3:waiting for process to exit 2026-02-22T22:12:03.008 INFO:teuthology.orchestra.run:waiting for 300 2026-02-22T22:12:03.033 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial026.stderr:2026-02-22T22:12:03.032+0000 7f45719e6640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.3 (PID: 83585) UID: 1000 2026-02-22T22:12:03.036 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial026.stderr:2026-02-22T22:12:03.032+0000 7f456a9d8640 -1 rbd::mirror::image_deleter::TrashWatcher: 0x560f1cb9d440 handle_unregister_watcher: error unregistering watcher for trash directory: (2) No such file or directory 2026-02-22T22:12:03.036 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial026.stderr:2026-02-22T22:12:03.032+0000 7f455c9bc640 -1 rbd::mirror::image_deleter::TrashWatcher: 0x560f1c8c6000 handle_unregister_watcher: error unregistering watcher for trash directory: (2) No such file or directory 2026-02-22T22:12:03.037 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial026.stderr:2026-02-22T22:12:03.032+0000 7f456a9d8640 -1 rbd::mirror::image_deleter::TrashWatcher: 0x560f1cb9db00 handle_unregister_watcher: error unregistering watcher for trash directory: (2) No such file or directory 2026-02-22T22:12:03.037 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial026.stderr:2026-02-22T22:12:03.032+0000 7f456a1d7640 -1 rbd::mirror::PoolWatcher: 0x560f1cc841e0 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-02-22T22:12:03.037 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial026.stderr:2026-02-22T22:12:03.032+0000 7f456a9d8640 -1 rbd::mirror::PoolWatcher: 0x560f1cc843c0 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-02-22T22:12:03.037 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial026.stderr:2026-02-22T22:12:03.032+0000 7f45641cb640 -1 rbd::mirror::PoolWatcher: 0x560f1ccb61e0 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-02-22T22:12:03.037 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial026.stderr:2026-02-22T22:12:03.032+0000 7f45641cb640 -1 rbd::mirror::PoolWatcher: 0x560f1cc845a0 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-02-22T22:12:03.037 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial026.stderr:2026-02-22T22:12:03.032+0000 7f455d1bd640 -1 rbd::mirror::PoolWatcher: 0x560f1c8bd860 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-02-22T22:12:03.037 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial026.stderr:2026-02-22T22:12:03.033+0000 7f45711e5640 -1 rbd::mirror::InstanceWatcher: 0x560f1c819180 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5867 object: (2) No such file or directory 2026-02-22T22:12:03.037 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial026.stderr:2026-02-22T22:12:03.033+0000 7f45709e4640 -1 rbd::mirror::InstanceWatcher: 0x560f1cf73880 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5867 object: (2) No such file or directory 2026-02-22T22:12:03.037 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial026.stderr:2026-02-22T22:12:03.033+0000 7f456a1d7640 -1 rbd::mirror::InstanceWatcher: 0x560f1c819180 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-22T22:12:03.037 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial026.stderr:2026-02-22T22:12:03.033+0000 7f456a9d8640 -1 rbd::mirror::InstanceWatcher: 0x560f1cf73880 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-22T22:12:03.038 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial026.stderr:2026-02-22T22:12:03.033+0000 7f45641cb640 -1 rbd::mirror::MirrorStatusUpdater 0x560f1c7e2360 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-22T22:12:03.038 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial026.stderr:2026-02-22T22:12:03.033+0000 7f45709e4640 -1 rbd::mirror::NamespaceReplayer: 0x560f1c7e01a0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-22T22:12:03.038 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial026.stderr:2026-02-22T22:12:03.033+0000 7f45639ca640 -1 rbd::mirror::MirrorStatusUpdater 0x560f1c7e25a0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-22T22:12:03.038 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial026.stderr:2026-02-22T22:12:03.033+0000 7f45711e5640 -1 rbd::mirror::NamespaceReplayer: 0x560f1c7e0000 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-22T22:12:03.038 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial026.stderr:2026-02-22T22:12:03.033+0000 7f456a1d7640 -1 rbd::mirror::MirrorStatusUpdater 0x560f1c7e2120 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-22T22:12:03.038 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial026.stderr:2026-02-22T22:12:03.033+0000 7f45709e4640 -1 rbd::mirror::NamespaceReplayer: 0x560f1c7e01a0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-22T22:12:03.038 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial026.stderr:2026-02-22T22:12:03.033+0000 7f456a9d8640 -1 rbd::mirror::MirrorStatusUpdater 0x560f1c7e2000 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-22T22:12:03.038 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial026.stderr:2026-02-22T22:12:03.033+0000 7f45709e4640 -1 rbd::mirror::NamespaceReplayer: 0x560f1c7e0000 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-22T22:12:03.038 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial026.stderr:2026-02-22T22:12:03.033+0000 7f45711e5640 -1 rbd::mirror::InstanceWatcher: 0x560f1cbb0380 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5870 object: (2) No such file or directory 2026-02-22T22:12:03.038 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial026.stderr:2026-02-22T22:12:03.034+0000 7f455d1bd640 -1 rbd::mirror::InstanceWatcher: 0x560f1cbb0380 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-22T22:12:03.039 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial026.stderr:2026-02-22T22:12:03.034+0000 7f455c9bc640 -1 rbd::mirror::MirrorStatusUpdater 0x560f1c9e30e0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-22T22:12:03.039 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial026.stderr:2026-02-22T22:12:03.034+0000 7f45709e4640 -1 rbd::mirror::NamespaceReplayer: 0x560f1cc524e0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-22T22:12:03.039 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial026.stderr:2026-02-22T22:12:03.035+0000 7f455d1bd640 -1 rbd::mirror::image_deleter::TrashWatcher: 0x560f1c74dd40 handle_unregister_watcher: error unregistering watcher for trash directory: (2) No such file or directory 2026-02-22T22:12:03.039 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial026.stderr:2026-02-22T22:12:03.035+0000 7f455c9bc640 -1 rbd::mirror::PoolWatcher: 0x560f1ccb65a0 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-02-22T22:12:03.041 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial026.stderr:2026-02-22T22:12:03.035+0000 7f45561af640 -1 rbd::mirror::PoolWatcher: 0x560f1ccb6960 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-02-22T22:12:03.042 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial026.stderr:2026-02-22T22:12:03.035+0000 7f45709e4640 -1 rbd::mirror::LeaderWatcher: 0x560f1caf2300 handle_notify_lock_released: error notifying leader lock released: (2) No such file or directory 2026-02-22T22:12:03.042 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial026.stderr:2026-02-22T22:12:03.035+0000 7f45709e4640 -1 rbd::mirror::LeaderWatcher: 0x560f1caf2300 handle_shut_down_leader_lock: error shutting down leader lock: (2) No such file or directory 2026-02-22T22:12:03.042 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial026.stderr:2026-02-22T22:12:03.035+0000 7f45711e5640 -1 rbd::mirror::LeaderWatcher: 0x560f1caf2300 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2026-02-22T22:12:03.042 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial026.stderr:2026-02-22T22:12:03.035+0000 7f45711e5640 -1 rbd::mirror::InstanceWatcher: 0x560f1cf73c00 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5870 object: (2) No such file or directory 2026-02-22T22:12:03.042 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial026.stderr:2026-02-22T22:12:03.036+0000 7f455c9bc640 -1 rbd::mirror::InstanceWatcher: 0x560f1cf73c00 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-22T22:12:03.042 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial026.stderr:2026-02-22T22:12:03.036+0000 7f45569b0640 -1 rbd::mirror::MirrorStatusUpdater 0x560f1c9e27e0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-22T22:12:03.042 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial026.stderr:2026-02-22T22:12:03.036+0000 7f45709e4640 -1 rbd::mirror::NamespaceReplayer: 0x560f1cc52000 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-22T22:12:03.042 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial026.stderr:2026-02-22T22:12:03.036+0000 7f455d1bd640 -1 rbd::mirror::MirrorStatusUpdater 0x560f1c9a78c0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-22T22:12:03.042 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial026.stderr:2026-02-22T22:12:03.036+0000 7f45711e5640 -1 rbd::mirror::NamespaceReplayer: 0x560f1cc52000 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-22T22:12:03.042 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial026.stderr:2026-02-22T22:12:03.037+0000 7f456a1d7640 -1 rbd::mirror::image_deleter::TrashWatcher: 0x560f1c7706c0 handle_unregister_watcher: error unregistering watcher for trash directory: (2) No such file or directory 2026-02-22T22:12:03.042 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial026.stderr:2026-02-22T22:12:03.037+0000 7f45641cb640 -1 rbd::mirror::PoolWatcher: 0x560f1cc84d20 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-02-22T22:12:03.043 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial026.stderr:2026-02-22T22:12:03.037+0000 7f456a9d8640 -1 rbd::mirror::PoolWatcher: 0x560f1cc84000 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-02-22T22:12:03.043 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial026.stderr:2026-02-22T22:12:03.038+0000 7f45711e5640 -1 rbd::mirror::LeaderWatcher: 0x560f1c51bb00 handle_notify_lock_released: error notifying leader lock released: (2) No such file or directory 2026-02-22T22:12:03.043 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial026.stderr:2026-02-22T22:12:03.038+0000 7f45711e5640 -1 rbd::mirror::LeaderWatcher: 0x560f1c51bb00 handle_shut_down_leader_lock: error shutting down leader lock: (2) No such file or directory 2026-02-22T22:12:03.043 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial026.stderr:2026-02-22T22:12:03.038+0000 7f45709e4640 -1 rbd::mirror::LeaderWatcher: 0x560f1c51bb00 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2026-02-22T22:12:03.043 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial026.stderr:2026-02-22T22:12:03.038+0000 7f45709e4640 -1 rbd::mirror::InstanceWatcher: 0x560f1cf73500 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5867 object: (2) No such file or directory 2026-02-22T22:12:03.043 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial026.stderr:2026-02-22T22:12:03.038+0000 7f456a9d8640 -1 rbd::mirror::InstanceWatcher: 0x560f1cf73500 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-22T22:12:03.043 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial026.stderr:2026-02-22T22:12:03.038+0000 7f45639ca640 -1 rbd::mirror::MirrorStatusUpdater 0x560f1c50c6c0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-22T22:12:03.043 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial026.stderr:2026-02-22T22:12:03.038+0000 7f45711e5640 -1 rbd::mirror::NamespaceReplayer: 0x560f1b5524e0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-22T22:12:03.043 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial026.stderr:2026-02-22T22:12:03.038+0000 7f456a1d7640 -1 rbd::mirror::MirrorStatusUpdater 0x560f1c497560 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-22T22:12:03.043 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial026.stderr:2026-02-22T22:12:03.038+0000 7f45709e4640 -1 rbd::mirror::NamespaceReplayer: 0x560f1b5524e0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-22T22:12:03.050 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Stopped 2026-02-22T22:12:03.050 DEBUG:teuthology.run_tasks:Unwinding manager rbd-mirror 2026-02-22T22:12:03.068 DEBUG:tasks.rbd_mirror.cluster1.client.mirror.2:waiting for process to exit 2026-02-22T22:12:03.068 INFO:teuthology.orchestra.run:waiting for 300 2026-02-22T22:12:03.090 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial026.stderr:2026-02-22T22:12:03.089+0000 7f80972f6640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.2 (PID: 84538) UID: 1000 2026-02-22T22:12:03.092 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial026.stderr:2026-02-22T22:12:03.089+0000 7f80962f4640 -1 rbd::mirror::InstanceWatcher: 0x55c89737f880 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5921 object: (2) No such file or directory 2026-02-22T22:12:03.092 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial026.stderr:2026-02-22T22:12:03.089+0000 7f8096af5640 -1 rbd::mirror::InstanceWatcher: 0x55c896c2b180 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5921 object: (2) No such file or directory 2026-02-22T22:12:03.092 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial026.stderr:2026-02-22T22:12:03.089+0000 7f80902e8640 -1 rbd::mirror::InstanceWatcher: 0x55c89737f880 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-22T22:12:03.092 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial026.stderr:2026-02-22T22:12:03.090+0000 7f808fae7640 -1 rbd::mirror::InstanceWatcher: 0x55c896c2b180 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-22T22:12:03.092 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial026.stderr:2026-02-22T22:12:03.090+0000 7f8089adb640 -1 rbd::mirror::MirrorStatusUpdater 0x55c896bf25a0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-22T22:12:03.092 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial026.stderr:2026-02-22T22:12:03.090+0000 7f8089adb640 -1 rbd::mirror::MirrorStatusUpdater 0x55c896bf2360 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-22T22:12:03.093 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial026.stderr:2026-02-22T22:12:03.090+0000 7f80962f4640 -1 rbd::mirror::NamespaceReplayer: 0x55c896bf01a0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-22T22:12:03.093 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial026.stderr:2026-02-22T22:12:03.090+0000 7f80962f4640 -1 rbd::mirror::NamespaceReplayer: 0x55c896bf0000 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-22T22:12:03.093 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial026.stderr:2026-02-22T22:12:03.090+0000 7f80902e8640 -1 rbd::mirror::MirrorStatusUpdater 0x55c896bf2120 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-22T22:12:03.093 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial026.stderr:2026-02-22T22:12:03.090+0000 7f8096af5640 -1 rbd::mirror::NamespaceReplayer: 0x55c896bf01a0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-22T22:12:03.093 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial026.stderr:2026-02-22T22:12:03.090+0000 7f808fae7640 -1 rbd::mirror::MirrorStatusUpdater 0x55c896bf2000 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-22T22:12:03.093 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial026.stderr:2026-02-22T22:12:03.090+0000 7f80962f4640 -1 rbd::mirror::NamespaceReplayer: 0x55c896bf0000 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-22T22:12:03.945 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial026.stderr:2026-02-22T22:12:03.944+0000 7f8096af5640 -1 rbd::mirror::InstanceWatcher: 0x55c896fbe700 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5924 object: (2) No such file or directory 2026-02-22T22:12:03.946 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial026.stderr:2026-02-22T22:12:03.944+0000 7f8082acd640 -1 rbd::mirror::InstanceWatcher: 0x55c896fbe700 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-22T22:12:03.947 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial026.stderr:2026-02-22T22:12:03.944+0000 7f807c2c0640 -1 rbd::mirror::MirrorStatusUpdater 0x55c896df4900 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-22T22:12:03.947 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial026.stderr:2026-02-22T22:12:03.944+0000 7f80962f4640 -1 rbd::mirror::NamespaceReplayer: 0x55c897082000 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-22T22:12:03.947 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial026.stderr:2026-02-22T22:12:03.945+0000 7f80822cc640 -1 rbd::mirror::InstanceWatcher: 0x55c896fbe380 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-22T22:12:03.947 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial026.stderr:2026-02-22T22:12:03.945+0000 7f807babf640 -1 rbd::mirror::MirrorStatusUpdater 0x55c896df47e0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-22T22:12:03.948 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial026.stderr:2026-02-22T22:12:03.945+0000 7f8096af5640 -1 rbd::mirror::NamespaceReplayer: 0x55c896bf04e0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-22T22:12:03.950 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial026.stderr:2026-02-22T22:12:03.948+0000 7f80962f4640 -1 rbd::mirror::LeaderWatcher: 0x55c896936c00 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2026-02-22T22:12:03.950 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial026.stderr:2026-02-22T22:12:03.949+0000 7f80962f4640 -1 rbd::mirror::InstanceWatcher: 0x55c89737f500 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5921 object: (2) No such file or directory 2026-02-22T22:12:03.950 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial026.stderr:2026-02-22T22:12:03.949+0000 7f80902e8640 -1 rbd::mirror::InstanceWatcher: 0x55c89737f500 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-22T22:12:03.951 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial026.stderr:2026-02-22T22:12:03.949+0000 7f80892da640 -1 rbd::mirror::MirrorStatusUpdater 0x55c896924240 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-22T22:12:03.951 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial026.stderr:2026-02-22T22:12:03.949+0000 7f80962f4640 -1 rbd::mirror::NamespaceReplayer: 0x55c89677da00 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-22T22:12:03.951 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial026.stderr:2026-02-22T22:12:03.949+0000 7f808fae7640 -1 rbd::mirror::MirrorStatusUpdater 0x55c8968b39e0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-22T22:12:03.951 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial026.stderr:2026-02-22T22:12:03.949+0000 7f8096af5640 -1 rbd::mirror::NamespaceReplayer: 0x55c89677da00 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-22T22:12:09.170 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Stopped 2026-02-22T22:12:09.170 DEBUG:teuthology.run_tasks:Unwinding manager rbd-mirror 2026-02-22T22:12:09.183 DEBUG:tasks.rbd_mirror.cluster1.client.mirror.1:waiting for process to exit 2026-02-22T22:12:09.183 INFO:teuthology.orchestra.run:waiting for 300 2026-02-22T22:12:09.183 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial026.stderr:2026-02-22T22:12:09.182+0000 7fc9f15db640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 (PID: 82171) UID: 1000 2026-02-22T22:12:09.184 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial026.stderr:2026-02-22T22:12:09.182+0000 7fc9f05d9640 -1 rbd::mirror::InstanceWatcher: 0x555ae6eea700 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5803 object: (2) No such file or directory 2026-02-22T22:12:09.184 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial026.stderr:2026-02-22T22:12:09.182+0000 7fc9f0dda640 -1 rbd::mirror::InstanceWatcher: 0x555ae758b880 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5800 object: (2) No such file or directory 2026-02-22T22:12:09.185 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial026.stderr:2026-02-22T22:12:09.182+0000 7fc9f05d9640 -1 rbd::mirror::InstanceWatcher: 0x555ae758bc00 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5800 object: (2) No such file or directory 2026-02-22T22:12:09.185 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial026.stderr:2026-02-22T22:12:09.183+0000 7fc9dcdb2640 -1 rbd::mirror::InstanceWatcher: 0x555ae6eea700 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-22T22:12:09.185 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial026.stderr:2026-02-22T22:12:09.183+0000 7fc9ea5cd640 -1 rbd::mirror::InstanceWatcher: 0x555ae758b880 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-22T22:12:09.185 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial026.stderr:2026-02-22T22:12:09.183+0000 7fc9e9dcc640 -1 rbd::mirror::InstanceWatcher: 0x555ae758bc00 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-22T22:12:09.185 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial026.stderr:2026-02-22T22:12:09.183+0000 7fc9e35bf640 -1 rbd::mirror::MirrorStatusUpdater 0x555ae6b0e360 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-22T22:12:09.185 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial026.stderr:2026-02-22T22:12:09.183+0000 7fc9f05d9640 -1 rbd::mirror::LeaderWatcher: 0x555ae6ec2300 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2026-02-22T22:12:09.185 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial026.stderr:2026-02-22T22:12:09.183+0000 7fc9f05d9640 -1 rbd::mirror::NamespaceReplayer: 0x555ae6b0c000 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-22T22:12:09.185 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial026.stderr:2026-02-22T22:12:09.183+0000 7fc9ea5cd640 -1 rbd::mirror::MirrorStatusUpdater 0x555ae6b0e000 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-22T22:12:09.185 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial026.stderr:2026-02-22T22:12:09.183+0000 7fc9f0dda640 -1 rbd::mirror::NamespaceReplayer: 0x555ae6b0c000 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-22T22:12:09.185 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial026.stderr:2026-02-22T22:12:09.183+0000 7fc9e9dcc640 -1 rbd::mirror::MirrorStatusUpdater 0x555ae6b0e120 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-22T22:12:09.185 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial026.stderr:2026-02-22T22:12:09.183+0000 7fc9f05d9640 -1 rbd::mirror::NamespaceReplayer: 0x555ae6b0c1a0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-22T22:12:09.185 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial026.stderr:2026-02-22T22:12:09.183+0000 7fc9f05d9640 -1 rbd::mirror::InstanceWatcher: 0x555ae6eea380 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5803 object: (2) No such file or directory 2026-02-22T22:12:09.186 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial026.stderr:2026-02-22T22:12:09.183+0000 7fc9dc5b1640 -1 rbd::mirror::InstanceWatcher: 0x555ae6eea380 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-22T22:12:09.186 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial026.stderr:2026-02-22T22:12:09.184+0000 7fc9d5da4640 -1 rbd::mirror::MirrorStatusUpdater 0x555ae6b0eb40 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-22T22:12:09.186 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial026.stderr:2026-02-22T22:12:09.184+0000 7fc9f05d9640 -1 rbd::mirror::NamespaceReplayer: 0x555ae6f8a000 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-22T22:12:09.186 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial026.stderr:2026-02-22T22:12:09.184+0000 7fc9dcdb2640 -1 rbd::mirror::MirrorStatusUpdater 0x555ae6cd1c20 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-22T22:12:09.186 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial026.stderr:2026-02-22T22:12:09.184+0000 7fc9f0dda640 -1 rbd::mirror::NamespaceReplayer: 0x555ae6f8a000 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-22T22:12:09.187 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial026.stderr:2026-02-22T22:12:09.185+0000 7fc9f0dda640 -1 rbd::mirror::LeaderWatcher: 0x555ae7731b00 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2026-02-22T22:12:09.187 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial026.stderr:2026-02-22T22:12:09.186+0000 7fc9f0dda640 -1 rbd::mirror::InstanceWatcher: 0x555ae758b500 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5800 object: (2) No such file or directory 2026-02-22T22:12:09.187 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial026.stderr:2026-02-22T22:12:09.186+0000 7fc9ea5cd640 -1 rbd::mirror::InstanceWatcher: 0x555ae758b500 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-22T22:12:09.187 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial026.stderr:2026-02-22T22:12:09.186+0000 7fc9e3dc0640 -1 rbd::mirror::MirrorStatusUpdater 0x555ae771c6c0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-22T22:12:09.188 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial026.stderr:2026-02-22T22:12:09.186+0000 7fc9f05d9640 -1 rbd::mirror::NamespaceReplayer: 0x555ae6969d40 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-22T22:12:09.188 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial026.stderr:2026-02-22T22:12:09.186+0000 7fc9e9dcc640 -1 rbd::mirror::MirrorStatusUpdater 0x555ae76a7560 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-22T22:12:09.188 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial026.stderr:2026-02-22T22:12:09.186+0000 7fc9f0dda640 -1 rbd::mirror::NamespaceReplayer: 0x555ae6969d40 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-22T22:12:09.197 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Stopped 2026-02-22T22:12:09.197 DEBUG:teuthology.run_tasks:Unwinding manager rbd-mirror 2026-02-22T22:12:09.209 DEBUG:tasks.rbd_mirror.cluster1.client.mirror.0:waiting for process to exit 2026-02-22T22:12:09.209 INFO:teuthology.orchestra.run:waiting for 300 2026-02-22T22:12:09.238 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial026.stderr:2026-02-22T22:12:09.237+0000 7f23959be640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 (PID: 64898) UID: 1000 2026-02-22T22:12:09.239 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial026.stderr:2026-02-22T22:12:09.237+0000 7f23951bd640 -1 rbd::mirror::InstanceWatcher: 0x563429a7a380 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.4904 object: (2) No such file or directory 2026-02-22T22:12:09.239 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial026.stderr:2026-02-22T22:12:09.237+0000 7f23949bc640 -1 rbd::mirror::InstanceWatcher: 0x563429e3d880 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.4902 object: (2) No such file or directory 2026-02-22T22:12:09.239 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial026.stderr:2026-02-22T22:12:09.237+0000 7f23951bd640 -1 rbd::mirror::InstanceWatcher: 0x563429719500 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.4902 object: (2) No such file or directory 2026-02-22T22:12:09.239 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial026.stderr:2026-02-22T22:12:09.237+0000 7f2381195640 -1 rbd::mirror::InstanceWatcher: 0x563429a7a380 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-22T22:12:09.239 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial026.stderr:2026-02-22T22:12:09.238+0000 7f238e9b0640 -1 rbd::mirror::InstanceWatcher: 0x563429e3d880 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-22T22:12:09.239 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial026.stderr:2026-02-22T22:12:09.238+0000 7f23949bc640 -1 rbd::mirror::InstanceWatcher: 0x563429719880 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.4904 object: (2) No such file or directory 2026-02-22T22:12:09.239 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial026.stderr:2026-02-22T22:12:09.238+0000 7f238e1af640 -1 rbd::mirror::InstanceWatcher: 0x563429719500 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-22T22:12:09.239 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial026.stderr:2026-02-22T22:12:09.238+0000 7f23879a2640 -1 rbd::mirror::MirrorStatusUpdater 0x5634296ae5a0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-22T22:12:09.240 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial026.stderr:2026-02-22T22:12:09.238+0000 7f23949bc640 -1 rbd::mirror::NamespaceReplayer: 0x5634296ac000 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-22T22:12:09.240 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial026.stderr:2026-02-22T22:12:09.238+0000 7f23881a3640 -1 rbd::mirror::MirrorStatusUpdater 0x5634296ae360 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-22T22:12:09.240 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial026.stderr:2026-02-22T22:12:09.238+0000 7f23951bd640 -1 rbd::mirror::NamespaceReplayer: 0x5634296ac1a0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-22T22:12:09.240 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial026.stderr:2026-02-22T22:12:09.238+0000 7f2381195640 -1 rbd::mirror::InstanceWatcher: 0x563429719880 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-22T22:12:09.240 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial026.stderr:2026-02-22T22:12:09.238+0000 7f238e9b0640 -1 rbd::mirror::MirrorStatusUpdater 0x5634296ae000 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-22T22:12:09.240 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial026.stderr:2026-02-22T22:12:09.238+0000 7f23951bd640 -1 rbd::mirror::NamespaceReplayer: 0x5634296ac000 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-22T22:12:09.240 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial026.stderr:2026-02-22T22:12:09.238+0000 7f237a988640 -1 rbd::mirror::MirrorStatusUpdater 0x5634298a67e0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-22T22:12:09.240 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial026.stderr:2026-02-22T22:12:09.238+0000 7f23951bd640 -1 rbd::mirror::NamespaceReplayer: 0x5634296ac4e0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-22T22:12:09.240 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial026.stderr:2026-02-22T22:12:09.238+0000 7f238e1af640 -1 rbd::mirror::MirrorStatusUpdater 0x5634296ae120 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-22T22:12:09.240 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial026.stderr:2026-02-22T22:12:09.238+0000 7f23951bd640 -1 rbd::mirror::NamespaceReplayer: 0x5634296ac1a0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-22T22:12:09.241 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial026.stderr:2026-02-22T22:12:09.240+0000 7f23949bc640 -1 rbd::mirror::LeaderWatcher: 0x5634293e5b00 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2026-02-22T22:12:09.242 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial026.stderr:2026-02-22T22:12:09.240+0000 7f23949bc640 -1 rbd::mirror::InstanceWatcher: 0x563429e3d500 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.4902 object: (2) No such file or directory 2026-02-22T22:12:09.242 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial026.stderr:2026-02-22T22:12:09.240+0000 7f238e9b0640 -1 rbd::mirror::InstanceWatcher: 0x563429e3d500 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-22T22:12:09.242 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial026.stderr:2026-02-22T22:12:09.241+0000 7f23879a2640 -1 rbd::mirror::MirrorStatusUpdater 0x5634293d26c0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-22T22:12:09.242 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial026.stderr:2026-02-22T22:12:09.241+0000 7f23949bc640 -1 rbd::mirror::NamespaceReplayer: 0x5634291c41a0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-22T22:12:09.242 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial026.stderr:2026-02-22T22:12:09.241+0000 7f238e1af640 -1 rbd::mirror::MirrorStatusUpdater 0x56342935d560 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-22T22:12:09.242 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial026.stderr:2026-02-22T22:12:09.241+0000 7f23951bd640 -1 rbd::mirror::NamespaceReplayer: 0x5634291c41a0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-22T22:12:09.255 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Stopped 2026-02-22T22:12:09.255 DEBUG:teuthology.run_tasks:Unwinding manager ceph 2026-02-22T22:12:09.266 INFO:tasks.ceph.ceph_manager.cluster2:waiting for clean 2026-02-22T22:12:09.267 DEBUG:teuthology.orchestra.run.trial026:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2026-02-22T22:12:09.453 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T22:12:09.454 INFO:teuthology.orchestra.run.trial026.stderr:dumped all 2026-02-22T22:12:09.462 INFO:teuthology.orchestra.run.trial026.stdout:{"pg_ready":true,"pg_map":{"version":1513,"stamp":"2026-02-22T22:12:09.225955+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":148,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":2906892,"kb_used_data":15888,"kb_used_omap":297,"kb_used_meta":2890646,"kb_avail":280208628,"statfs":{"total":289910292480,"available":286933635072,"internally_reserved":0,"allocated":16269312,"data_stored":9094327,"data_compressed":19632,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":304727,"internal_metadata":2960021929},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-192938022,"num_objects":-89,"num_object_clones":0,"num_object_copies":-178,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-89,"num_whiteouts":0,"num_read":-113560,"num_read_kb":-39588847,"num_write":-1502822,"num_write_kb":-563039732,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":-17,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.001044"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":907,"reported_epoch":454,"state":"active+clean","last_fresh":"2026-02-22T22:12:02.255700+0000","last_change":"2026-02-22T21:36:57.117998+0000","last_active":"2026-02-22T22:12:02.255700+0000","last_peered":"2026-02-22T22:12:02.255700+0000","last_clean":"2026-02-22T22:12:02.255700+0000","last_became_active":"2026-02-22T21:36:55.116298+0000","last_became_peered":"2026-02-22T21:36:55.116298+0000","last_unstale":"2026-02-22T22:12:02.255700+0000","last_undegraded":"2026-02-22T22:12:02.255700+0000","last_fullsized":"2026-02-22T22:12:02.255700+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:54.106037+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:54.106037+0000","last_clean_scrub_stamp":"2026-02-22T21:36:54.106037+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T02:34:53.292431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00037892999999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":907,"reported_epoch":454,"state":"active+clean","last_fresh":"2026-02-22T22:12:02.258275+0000","last_change":"2026-02-22T21:36:57.118108+0000","last_active":"2026-02-22T22:12:02.258275+0000","last_peered":"2026-02-22T22:12:02.258275+0000","last_clean":"2026-02-22T22:12:02.258275+0000","last_became_active":"2026-02-22T21:36:55.116070+0000","last_became_peered":"2026-02-22T21:36:55.116070+0000","last_unstale":"2026-02-22T22:12:02.258275+0000","last_undegraded":"2026-02-22T22:12:02.258275+0000","last_fullsized":"2026-02-22T22:12:02.258275+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:54.106037+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:54.106037+0000","last_clean_scrub_stamp":"2026-02-22T21:36:54.106037+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T07:44:09.720355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048051700000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":907,"reported_epoch":454,"state":"active+clean","last_fresh":"2026-02-22T22:12:02.259437+0000","last_change":"2026-02-22T21:36:57.118060+0000","last_active":"2026-02-22T22:12:02.259437+0000","last_peered":"2026-02-22T22:12:02.259437+0000","last_clean":"2026-02-22T22:12:02.259437+0000","last_became_active":"2026-02-22T21:36:55.116352+0000","last_became_peered":"2026-02-22T21:36:55.116352+0000","last_unstale":"2026-02-22T22:12:02.259437+0000","last_undegraded":"2026-02-22T22:12:02.259437+0000","last_fullsized":"2026-02-22T22:12:02.259437+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:54.106037+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:54.106037+0000","last_clean_scrub_stamp":"2026-02-22T21:36:54.106037+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T00:50:23.097350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00044389899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":907,"reported_epoch":454,"state":"active+clean","last_fresh":"2026-02-22T22:12:02.253765+0000","last_change":"2026-02-22T21:36:57.117889+0000","last_active":"2026-02-22T22:12:02.253765+0000","last_peered":"2026-02-22T22:12:02.253765+0000","last_clean":"2026-02-22T22:12:02.253765+0000","last_became_active":"2026-02-22T21:36:55.116352+0000","last_became_peered":"2026-02-22T21:36:55.116352+0000","last_unstale":"2026-02-22T22:12:02.253765+0000","last_undegraded":"2026-02-22T22:12:02.253765+0000","last_fullsized":"2026-02-22T22:12:02.253765+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:54.106037+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:54.106037+0000","last_clean_scrub_stamp":"2026-02-22T21:36:54.106037+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T06:10:01.562314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029249000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"16'2","reported_seq":912,"reported_epoch":454,"state":"active+clean","last_fresh":"2026-02-22T22:12:03.213071+0000","last_change":"2026-02-22T21:36:57.116752+0000","last_active":"2026-02-22T22:12:03.213071+0000","last_peered":"2026-02-22T22:12:03.213071+0000","last_clean":"2026-02-22T22:12:03.213071+0000","last_became_active":"2026-02-22T21:36:55.116275+0000","last_became_peered":"2026-02-22T21:36:55.116275+0000","last_unstale":"2026-02-22T22:12:03.213071+0000","last_undegraded":"2026-02-22T22:12:03.213071+0000","last_fullsized":"2026-02-22T22:12:03.213071+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:54.106037+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:54.106037+0000","last_clean_scrub_stamp":"2026-02-22T21:36:54.106037+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T07:41:58.739291+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000424664,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":908,"reported_epoch":454,"state":"active+clean","last_fresh":"2026-02-22T22:12:03.251863+0000","last_change":"2026-02-22T21:36:57.115505+0000","last_active":"2026-02-22T22:12:03.251863+0000","last_peered":"2026-02-22T22:12:03.251863+0000","last_clean":"2026-02-22T22:12:03.251863+0000","last_became_active":"2026-02-22T21:36:55.116210+0000","last_became_peered":"2026-02-22T21:36:55.116210+0000","last_unstale":"2026-02-22T22:12:03.251863+0000","last_undegraded":"2026-02-22T22:12:03.251863+0000","last_fullsized":"2026-02-22T22:12:03.251863+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:54.106037+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:54.106037+0000","last_clean_scrub_stamp":"2026-02-22T21:36:54.106037+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T02:21:43.966839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024309900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":909,"reported_epoch":454,"state":"active+clean","last_fresh":"2026-02-22T22:12:03.251313+0000","last_change":"2026-02-22T21:36:57.115504+0000","last_active":"2026-02-22T22:12:03.251313+0000","last_peered":"2026-02-22T22:12:03.251313+0000","last_clean":"2026-02-22T22:12:03.251313+0000","last_became_active":"2026-02-22T21:36:55.116216+0000","last_became_peered":"2026-02-22T21:36:55.116216+0000","last_unstale":"2026-02-22T22:12:03.251313+0000","last_undegraded":"2026-02-22T22:12:03.251313+0000","last_fullsized":"2026-02-22T22:12:03.251313+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:54.106037+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:54.106037+0000","last_clean_scrub_stamp":"2026-02-22T21:36:54.106037+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T23:54:52.644119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000245073,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"14'1","reported_seq":908,"reported_epoch":454,"state":"active+clean","last_fresh":"2026-02-22T22:12:02.257778+0000","last_change":"2026-02-22T21:36:57.117874+0000","last_active":"2026-02-22T22:12:02.257778+0000","last_peered":"2026-02-22T22:12:02.257778+0000","last_clean":"2026-02-22T22:12:02.257778+0000","last_became_active":"2026-02-22T21:36:55.116006+0000","last_became_peered":"2026-02-22T21:36:55.116006+0000","last_unstale":"2026-02-22T22:12:02.257778+0000","last_undegraded":"2026-02-22T22:12:02.257778+0000","last_fullsized":"2026-02-22T22:12:02.257778+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:54.106037+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:54.106037+0000","last_clean_scrub_stamp":"2026-02-22T21:36:54.106037+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T03:58:14.400632+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029477300000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"11'72","reported_seq":999,"reported_epoch":454,"state":"active+clean","last_fresh":"2026-02-22T22:12:02.252737+0000","last_change":"2026-02-22T21:36:50.308369+0000","last_active":"2026-02-22T22:12:02.252737+0000","last_peered":"2026-02-22T22:12:02.252737+0000","last_clean":"2026-02-22T22:12:02.252737+0000","last_became_active":"2026-02-22T21:36:50.308099+0000","last_became_peered":"2026-02-22T21:36:50.308099+0000","last_unstale":"2026-02-22T22:12:02.252737+0000","last_undegraded":"2026-02-22T22:12:02.252737+0000","last_fullsized":"2026-02-22T22:12:02.252737+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:49.099350+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:49.099350+0000","last_clean_scrub_stamp":"2026-02-22T21:36:49.099350+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T09:35:05.112768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":1180736,"data_compressed":19632,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":1,"up_from":9,"seq":38654706091,"num_pgs":98,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":945048,"kb_used_data":5316,"kb_used_omap":76,"kb_used_meta":939635,"kb_avail":93426792,"statfs":{"total":96636764160,"available":95669035008,"internally_reserved":0,"allocated":5443584,"data_stored":3228238,"data_compressed":9816,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":78140,"internal_metadata":962186948},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":0,"up_from":9,"seq":38654706091,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1011032,"kb_used_data":5316,"kb_used_omap":126,"kb_used_meta":1005569,"kb_avail":93360808,"statfs":{"total":96636764160,"available":95601467392,"internally_reserved":0,"allocated":5443584,"data_stored":3228238,"data_compressed":9816,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":129039,"internal_metadata":1029703665},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":2,"up_from":8,"seq":34359738796,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":950812,"kb_used_data":5256,"kb_used_omap":95,"kb_used_meta":945440,"kb_avail":93421028,"statfs":{"total":96636764160,"available":95663132672,"internally_reserved":0,"allocated":5382144,"data_stored":2637851,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":97548,"internal_metadata":968131316},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":9816,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":9816,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-22T22:12:09.464 DEBUG:teuthology.orchestra.run.trial026:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2026-02-22T22:12:09.623 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T22:12:09.624 INFO:teuthology.orchestra.run.trial026.stderr:dumped all 2026-02-22T22:12:09.632 INFO:teuthology.orchestra.run.trial026.stdout:{"pg_ready":true,"pg_map":{"version":1513,"stamp":"2026-02-22T22:12:09.225955+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":148,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":2906892,"kb_used_data":15888,"kb_used_omap":297,"kb_used_meta":2890646,"kb_avail":280208628,"statfs":{"total":289910292480,"available":286933635072,"internally_reserved":0,"allocated":16269312,"data_stored":9094327,"data_compressed":19632,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":304727,"internal_metadata":2960021929},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-192938022,"num_objects":-89,"num_object_clones":0,"num_object_copies":-178,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-89,"num_whiteouts":0,"num_read":-113560,"num_read_kb":-39588847,"num_write":-1502822,"num_write_kb":-563039732,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":-17,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.001044"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":907,"reported_epoch":454,"state":"active+clean","last_fresh":"2026-02-22T22:12:02.255700+0000","last_change":"2026-02-22T21:36:57.117998+0000","last_active":"2026-02-22T22:12:02.255700+0000","last_peered":"2026-02-22T22:12:02.255700+0000","last_clean":"2026-02-22T22:12:02.255700+0000","last_became_active":"2026-02-22T21:36:55.116298+0000","last_became_peered":"2026-02-22T21:36:55.116298+0000","last_unstale":"2026-02-22T22:12:02.255700+0000","last_undegraded":"2026-02-22T22:12:02.255700+0000","last_fullsized":"2026-02-22T22:12:02.255700+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:54.106037+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:54.106037+0000","last_clean_scrub_stamp":"2026-02-22T21:36:54.106037+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T02:34:53.292431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00037892999999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":907,"reported_epoch":454,"state":"active+clean","last_fresh":"2026-02-22T22:12:02.258275+0000","last_change":"2026-02-22T21:36:57.118108+0000","last_active":"2026-02-22T22:12:02.258275+0000","last_peered":"2026-02-22T22:12:02.258275+0000","last_clean":"2026-02-22T22:12:02.258275+0000","last_became_active":"2026-02-22T21:36:55.116070+0000","last_became_peered":"2026-02-22T21:36:55.116070+0000","last_unstale":"2026-02-22T22:12:02.258275+0000","last_undegraded":"2026-02-22T22:12:02.258275+0000","last_fullsized":"2026-02-22T22:12:02.258275+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:54.106037+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:54.106037+0000","last_clean_scrub_stamp":"2026-02-22T21:36:54.106037+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T07:44:09.720355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048051700000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":907,"reported_epoch":454,"state":"active+clean","last_fresh":"2026-02-22T22:12:02.259437+0000","last_change":"2026-02-22T21:36:57.118060+0000","last_active":"2026-02-22T22:12:02.259437+0000","last_peered":"2026-02-22T22:12:02.259437+0000","last_clean":"2026-02-22T22:12:02.259437+0000","last_became_active":"2026-02-22T21:36:55.116352+0000","last_became_peered":"2026-02-22T21:36:55.116352+0000","last_unstale":"2026-02-22T22:12:02.259437+0000","last_undegraded":"2026-02-22T22:12:02.259437+0000","last_fullsized":"2026-02-22T22:12:02.259437+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:54.106037+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:54.106037+0000","last_clean_scrub_stamp":"2026-02-22T21:36:54.106037+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T00:50:23.097350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00044389899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":907,"reported_epoch":454,"state":"active+clean","last_fresh":"2026-02-22T22:12:02.253765+0000","last_change":"2026-02-22T21:36:57.117889+0000","last_active":"2026-02-22T22:12:02.253765+0000","last_peered":"2026-02-22T22:12:02.253765+0000","last_clean":"2026-02-22T22:12:02.253765+0000","last_became_active":"2026-02-22T21:36:55.116352+0000","last_became_peered":"2026-02-22T21:36:55.116352+0000","last_unstale":"2026-02-22T22:12:02.253765+0000","last_undegraded":"2026-02-22T22:12:02.253765+0000","last_fullsized":"2026-02-22T22:12:02.253765+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:54.106037+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:54.106037+0000","last_clean_scrub_stamp":"2026-02-22T21:36:54.106037+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T06:10:01.562314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029249000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"16'2","reported_seq":912,"reported_epoch":454,"state":"active+clean","last_fresh":"2026-02-22T22:12:03.213071+0000","last_change":"2026-02-22T21:36:57.116752+0000","last_active":"2026-02-22T22:12:03.213071+0000","last_peered":"2026-02-22T22:12:03.213071+0000","last_clean":"2026-02-22T22:12:03.213071+0000","last_became_active":"2026-02-22T21:36:55.116275+0000","last_became_peered":"2026-02-22T21:36:55.116275+0000","last_unstale":"2026-02-22T22:12:03.213071+0000","last_undegraded":"2026-02-22T22:12:03.213071+0000","last_fullsized":"2026-02-22T22:12:03.213071+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:54.106037+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:54.106037+0000","last_clean_scrub_stamp":"2026-02-22T21:36:54.106037+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T07:41:58.739291+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000424664,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":908,"reported_epoch":454,"state":"active+clean","last_fresh":"2026-02-22T22:12:03.251863+0000","last_change":"2026-02-22T21:36:57.115505+0000","last_active":"2026-02-22T22:12:03.251863+0000","last_peered":"2026-02-22T22:12:03.251863+0000","last_clean":"2026-02-22T22:12:03.251863+0000","last_became_active":"2026-02-22T21:36:55.116210+0000","last_became_peered":"2026-02-22T21:36:55.116210+0000","last_unstale":"2026-02-22T22:12:03.251863+0000","last_undegraded":"2026-02-22T22:12:03.251863+0000","last_fullsized":"2026-02-22T22:12:03.251863+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:54.106037+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:54.106037+0000","last_clean_scrub_stamp":"2026-02-22T21:36:54.106037+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T02:21:43.966839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024309900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":909,"reported_epoch":454,"state":"active+clean","last_fresh":"2026-02-22T22:12:03.251313+0000","last_change":"2026-02-22T21:36:57.115504+0000","last_active":"2026-02-22T22:12:03.251313+0000","last_peered":"2026-02-22T22:12:03.251313+0000","last_clean":"2026-02-22T22:12:03.251313+0000","last_became_active":"2026-02-22T21:36:55.116216+0000","last_became_peered":"2026-02-22T21:36:55.116216+0000","last_unstale":"2026-02-22T22:12:03.251313+0000","last_undegraded":"2026-02-22T22:12:03.251313+0000","last_fullsized":"2026-02-22T22:12:03.251313+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:54.106037+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:54.106037+0000","last_clean_scrub_stamp":"2026-02-22T21:36:54.106037+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T23:54:52.644119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000245073,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"14'1","reported_seq":908,"reported_epoch":454,"state":"active+clean","last_fresh":"2026-02-22T22:12:02.257778+0000","last_change":"2026-02-22T21:36:57.117874+0000","last_active":"2026-02-22T22:12:02.257778+0000","last_peered":"2026-02-22T22:12:02.257778+0000","last_clean":"2026-02-22T22:12:02.257778+0000","last_became_active":"2026-02-22T21:36:55.116006+0000","last_became_peered":"2026-02-22T21:36:55.116006+0000","last_unstale":"2026-02-22T22:12:02.257778+0000","last_undegraded":"2026-02-22T22:12:02.257778+0000","last_fullsized":"2026-02-22T22:12:02.257778+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:54.106037+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:54.106037+0000","last_clean_scrub_stamp":"2026-02-22T21:36:54.106037+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T03:58:14.400632+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029477300000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"11'72","reported_seq":999,"reported_epoch":454,"state":"active+clean","last_fresh":"2026-02-22T22:12:02.252737+0000","last_change":"2026-02-22T21:36:50.308369+0000","last_active":"2026-02-22T22:12:02.252737+0000","last_peered":"2026-02-22T22:12:02.252737+0000","last_clean":"2026-02-22T22:12:02.252737+0000","last_became_active":"2026-02-22T21:36:50.308099+0000","last_became_peered":"2026-02-22T21:36:50.308099+0000","last_unstale":"2026-02-22T22:12:02.252737+0000","last_undegraded":"2026-02-22T22:12:02.252737+0000","last_fullsized":"2026-02-22T22:12:02.252737+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:49.099350+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:49.099350+0000","last_clean_scrub_stamp":"2026-02-22T21:36:49.099350+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T09:35:05.112768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":1180736,"data_compressed":19632,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":1,"up_from":9,"seq":38654706091,"num_pgs":98,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":945048,"kb_used_data":5316,"kb_used_omap":76,"kb_used_meta":939635,"kb_avail":93426792,"statfs":{"total":96636764160,"available":95669035008,"internally_reserved":0,"allocated":5443584,"data_stored":3228238,"data_compressed":9816,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":78140,"internal_metadata":962186948},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":0,"up_from":9,"seq":38654706091,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1011032,"kb_used_data":5316,"kb_used_omap":126,"kb_used_meta":1005569,"kb_avail":93360808,"statfs":{"total":96636764160,"available":95601467392,"internally_reserved":0,"allocated":5443584,"data_stored":3228238,"data_compressed":9816,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":129039,"internal_metadata":1029703665},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":2,"up_from":8,"seq":34359738796,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":950812,"kb_used_data":5256,"kb_used_omap":95,"kb_used_meta":945440,"kb_avail":93421028,"statfs":{"total":96636764160,"available":95663132672,"internally_reserved":0,"allocated":5382144,"data_stored":2637851,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":97548,"internal_metadata":968131316},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":9816,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":9816,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-22T22:12:09.633 INFO:tasks.ceph.ceph_manager.cluster2:clean! 2026-02-22T22:12:09.634 DEBUG:teuthology.orchestra.run.trial026:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2026-02-22T22:12:09.904 INFO:teuthology.orchestra.run.trial026.stderr:2026-02-22T22:12:09.902+0000 7f955effd640 0 --2- 10.20.193.26:0/1368147948 >> [v2:10.20.193.26:3300/0,v1:10.20.193.26:6789/0] conn(0x7f956006c890 0x7f95601471e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request get_initial_auth_request returned -2 2026-02-22T22:12:09.992 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T22:12:09.993 INFO:teuthology.orchestra.run.trial026.stderr:dumped all 2026-02-22T22:12:10.000 INFO:teuthology.orchestra.run.trial026.stdout:{"pg_ready":true,"pg_map":{"version":1513,"stamp":"2026-02-22T22:12:09.225955+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":148,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":2906892,"kb_used_data":15888,"kb_used_omap":297,"kb_used_meta":2890646,"kb_avail":280208628,"statfs":{"total":289910292480,"available":286933635072,"internally_reserved":0,"allocated":16269312,"data_stored":9094327,"data_compressed":19632,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":304727,"internal_metadata":2960021929},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-192938022,"num_objects":-89,"num_object_clones":0,"num_object_copies":-178,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-89,"num_whiteouts":0,"num_read":-113560,"num_read_kb":-39588847,"num_write":-1502822,"num_write_kb":-563039732,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":-17,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.001044"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":907,"reported_epoch":454,"state":"active+clean","last_fresh":"2026-02-22T22:12:02.255700+0000","last_change":"2026-02-22T21:36:57.117998+0000","last_active":"2026-02-22T22:12:02.255700+0000","last_peered":"2026-02-22T22:12:02.255700+0000","last_clean":"2026-02-22T22:12:02.255700+0000","last_became_active":"2026-02-22T21:36:55.116298+0000","last_became_peered":"2026-02-22T21:36:55.116298+0000","last_unstale":"2026-02-22T22:12:02.255700+0000","last_undegraded":"2026-02-22T22:12:02.255700+0000","last_fullsized":"2026-02-22T22:12:02.255700+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:54.106037+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:54.106037+0000","last_clean_scrub_stamp":"2026-02-22T21:36:54.106037+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T02:34:53.292431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00037892999999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":907,"reported_epoch":454,"state":"active+clean","last_fresh":"2026-02-22T22:12:02.258275+0000","last_change":"2026-02-22T21:36:57.118108+0000","last_active":"2026-02-22T22:12:02.258275+0000","last_peered":"2026-02-22T22:12:02.258275+0000","last_clean":"2026-02-22T22:12:02.258275+0000","last_became_active":"2026-02-22T21:36:55.116070+0000","last_became_peered":"2026-02-22T21:36:55.116070+0000","last_unstale":"2026-02-22T22:12:02.258275+0000","last_undegraded":"2026-02-22T22:12:02.258275+0000","last_fullsized":"2026-02-22T22:12:02.258275+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:54.106037+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:54.106037+0000","last_clean_scrub_stamp":"2026-02-22T21:36:54.106037+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T07:44:09.720355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048051700000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":907,"reported_epoch":454,"state":"active+clean","last_fresh":"2026-02-22T22:12:02.259437+0000","last_change":"2026-02-22T21:36:57.118060+0000","last_active":"2026-02-22T22:12:02.259437+0000","last_peered":"2026-02-22T22:12:02.259437+0000","last_clean":"2026-02-22T22:12:02.259437+0000","last_became_active":"2026-02-22T21:36:55.116352+0000","last_became_peered":"2026-02-22T21:36:55.116352+0000","last_unstale":"2026-02-22T22:12:02.259437+0000","last_undegraded":"2026-02-22T22:12:02.259437+0000","last_fullsized":"2026-02-22T22:12:02.259437+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:54.106037+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:54.106037+0000","last_clean_scrub_stamp":"2026-02-22T21:36:54.106037+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T00:50:23.097350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00044389899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":907,"reported_epoch":454,"state":"active+clean","last_fresh":"2026-02-22T22:12:02.253765+0000","last_change":"2026-02-22T21:36:57.117889+0000","last_active":"2026-02-22T22:12:02.253765+0000","last_peered":"2026-02-22T22:12:02.253765+0000","last_clean":"2026-02-22T22:12:02.253765+0000","last_became_active":"2026-02-22T21:36:55.116352+0000","last_became_peered":"2026-02-22T21:36:55.116352+0000","last_unstale":"2026-02-22T22:12:02.253765+0000","last_undegraded":"2026-02-22T22:12:02.253765+0000","last_fullsized":"2026-02-22T22:12:02.253765+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:54.106037+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:54.106037+0000","last_clean_scrub_stamp":"2026-02-22T21:36:54.106037+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T06:10:01.562314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029249000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"16'2","reported_seq":912,"reported_epoch":454,"state":"active+clean","last_fresh":"2026-02-22T22:12:03.213071+0000","last_change":"2026-02-22T21:36:57.116752+0000","last_active":"2026-02-22T22:12:03.213071+0000","last_peered":"2026-02-22T22:12:03.213071+0000","last_clean":"2026-02-22T22:12:03.213071+0000","last_became_active":"2026-02-22T21:36:55.116275+0000","last_became_peered":"2026-02-22T21:36:55.116275+0000","last_unstale":"2026-02-22T22:12:03.213071+0000","last_undegraded":"2026-02-22T22:12:03.213071+0000","last_fullsized":"2026-02-22T22:12:03.213071+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:54.106037+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:54.106037+0000","last_clean_scrub_stamp":"2026-02-22T21:36:54.106037+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T07:41:58.739291+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000424664,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":908,"reported_epoch":454,"state":"active+clean","last_fresh":"2026-02-22T22:12:03.251863+0000","last_change":"2026-02-22T21:36:57.115505+0000","last_active":"2026-02-22T22:12:03.251863+0000","last_peered":"2026-02-22T22:12:03.251863+0000","last_clean":"2026-02-22T22:12:03.251863+0000","last_became_active":"2026-02-22T21:36:55.116210+0000","last_became_peered":"2026-02-22T21:36:55.116210+0000","last_unstale":"2026-02-22T22:12:03.251863+0000","last_undegraded":"2026-02-22T22:12:03.251863+0000","last_fullsized":"2026-02-22T22:12:03.251863+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:54.106037+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:54.106037+0000","last_clean_scrub_stamp":"2026-02-22T21:36:54.106037+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T02:21:43.966839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024309900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":909,"reported_epoch":454,"state":"active+clean","last_fresh":"2026-02-22T22:12:03.251313+0000","last_change":"2026-02-22T21:36:57.115504+0000","last_active":"2026-02-22T22:12:03.251313+0000","last_peered":"2026-02-22T22:12:03.251313+0000","last_clean":"2026-02-22T22:12:03.251313+0000","last_became_active":"2026-02-22T21:36:55.116216+0000","last_became_peered":"2026-02-22T21:36:55.116216+0000","last_unstale":"2026-02-22T22:12:03.251313+0000","last_undegraded":"2026-02-22T22:12:03.251313+0000","last_fullsized":"2026-02-22T22:12:03.251313+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:54.106037+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:54.106037+0000","last_clean_scrub_stamp":"2026-02-22T21:36:54.106037+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T23:54:52.644119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000245073,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"14'1","reported_seq":908,"reported_epoch":454,"state":"active+clean","last_fresh":"2026-02-22T22:12:02.257778+0000","last_change":"2026-02-22T21:36:57.117874+0000","last_active":"2026-02-22T22:12:02.257778+0000","last_peered":"2026-02-22T22:12:02.257778+0000","last_clean":"2026-02-22T22:12:02.257778+0000","last_became_active":"2026-02-22T21:36:55.116006+0000","last_became_peered":"2026-02-22T21:36:55.116006+0000","last_unstale":"2026-02-22T22:12:02.257778+0000","last_undegraded":"2026-02-22T22:12:02.257778+0000","last_fullsized":"2026-02-22T22:12:02.257778+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:54.106037+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:54.106037+0000","last_clean_scrub_stamp":"2026-02-22T21:36:54.106037+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T03:58:14.400632+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029477300000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"11'72","reported_seq":999,"reported_epoch":454,"state":"active+clean","last_fresh":"2026-02-22T22:12:02.252737+0000","last_change":"2026-02-22T21:36:50.308369+0000","last_active":"2026-02-22T22:12:02.252737+0000","last_peered":"2026-02-22T22:12:02.252737+0000","last_clean":"2026-02-22T22:12:02.252737+0000","last_became_active":"2026-02-22T21:36:50.308099+0000","last_became_peered":"2026-02-22T21:36:50.308099+0000","last_unstale":"2026-02-22T22:12:02.252737+0000","last_undegraded":"2026-02-22T22:12:02.252737+0000","last_fullsized":"2026-02-22T22:12:02.252737+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:49.099350+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:49.099350+0000","last_clean_scrub_stamp":"2026-02-22T21:36:49.099350+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T09:35:05.112768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":1180736,"data_compressed":19632,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":1,"up_from":9,"seq":38654706091,"num_pgs":98,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":945048,"kb_used_data":5316,"kb_used_omap":76,"kb_used_meta":939635,"kb_avail":93426792,"statfs":{"total":96636764160,"available":95669035008,"internally_reserved":0,"allocated":5443584,"data_stored":3228238,"data_compressed":9816,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":78140,"internal_metadata":962186948},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":0,"up_from":9,"seq":38654706091,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1011032,"kb_used_data":5316,"kb_used_omap":126,"kb_used_meta":1005569,"kb_avail":93360808,"statfs":{"total":96636764160,"available":95601467392,"internally_reserved":0,"allocated":5443584,"data_stored":3228238,"data_compressed":9816,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":129039,"internal_metadata":1029703665},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":2,"up_from":8,"seq":34359738796,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":950812,"kb_used_data":5256,"kb_used_omap":95,"kb_used_meta":945440,"kb_avail":93421028,"statfs":{"total":96636764160,"available":95663132672,"internally_reserved":0,"allocated":5382144,"data_stored":2637851,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":97548,"internal_metadata":968131316},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":9816,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":9816,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-22T22:12:10.002 DEBUG:teuthology.orchestra.run.trial026:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd dump --format=json 2026-02-22T22:12:10.160 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T22:12:10.161 INFO:teuthology.orchestra.run.trial026.stdout:{"epoch":454,"fsid":"2e52b784-3eb2-4856-a6f6-868b97c4e845","created":"2026-02-22T21:36:39.062947+0000","modified":"2026-02-22T22:12:02.244450+0000","last_up_change":"2026-02-22T21:36:48.098894+0000","last_in_change":"2026-02-22T21:36:39.629207+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-02-22T21:36:48.933035+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2026-02-22T21:36:53.238866+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":16,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"5ebb4bdc-b6e6-4276-a34c-a27c8810395c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":24,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.26:6808","nonce":671216846},{"type":"v1","addr":"10.20.193.26:6809","nonce":671216846}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.26:6810","nonce":671216846},{"type":"v1","addr":"10.20.193.26:6811","nonce":671216846}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.26:6814","nonce":671216846},{"type":"v1","addr":"10.20.193.26:6815","nonce":671216846}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.26:6812","nonce":671216846},{"type":"v1","addr":"10.20.193.26:6813","nonce":671216846}]},"public_addr":"10.20.193.26:6809/671216846","cluster_addr":"10.20.193.26:6811/671216846","heartbeat_back_addr":"10.20.193.26:6815/671216846","heartbeat_front_addr":"10.20.193.26:6813/671216846","state":["exists","up"]},{"osd":1,"uuid":"24ae9b1a-c77d-483c-abb8-148e6becaaca","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":24,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.26:6816","nonce":4085277355},{"type":"v1","addr":"10.20.193.26:6817","nonce":4085277355}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.26:6818","nonce":4085277355},{"type":"v1","addr":"10.20.193.26:6819","nonce":4085277355}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.26:6822","nonce":4085277355},{"type":"v1","addr":"10.20.193.26:6823","nonce":4085277355}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.26:6820","nonce":4085277355},{"type":"v1","addr":"10.20.193.26:6821","nonce":4085277355}]},"public_addr":"10.20.193.26:6817/4085277355","cluster_addr":"10.20.193.26:6819/4085277355","heartbeat_back_addr":"10.20.193.26:6823/4085277355","heartbeat_front_addr":"10.20.193.26:6821/4085277355","state":["exists","up"]},{"osd":2,"uuid":"d5ca9f56-e467-4298-8a36-49cf123cc629","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":24,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.26:6800","nonce":4146597801},{"type":"v1","addr":"10.20.193.26:6801","nonce":4146597801}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.26:6802","nonce":4146597801},{"type":"v1","addr":"10.20.193.26:6803","nonce":4146597801}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.26:6806","nonce":4146597801},{"type":"v1","addr":"10.20.193.26:6807","nonce":4146597801}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.26:6804","nonce":4146597801},{"type":"v1","addr":"10.20.193.26:6805","nonce":4146597801}]},"public_addr":"10.20.193.26:6801/4146597801","cluster_addr":"10.20.193.26:6803/4146597801","heartbeat_back_addr":"10.20.193.26:6807/4146597801","heartbeat_front_addr":"10.20.193.26:6805/4146597801","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T21:36:46.235412+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T21:36:46.591328+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T21:36:46.061496+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"10.20.193.26:0/2194536449":"2026-02-22T23:11:59.077712+0000","10.20.193.26:0/1967698116":"2026-02-22T23:10:46.698832+0000","10.20.193.26:0/1425876245":"2026-02-22T23:10:10.962687+0000","10.20.193.26:0/2333150828":"2026-02-22T23:09:25.546087+0000","10.20.193.26:0/467007000":"2026-02-22T23:08:44.021290+0000","10.20.193.26:0/354331060":"2026-02-22T23:06:40.394652+0000","10.20.193.26:0/4064616147":"2026-02-22T23:07:17.257750+0000","10.20.193.26:0/2343192326":"2026-02-22T23:05:58.617332+0000","10.20.193.26:0/552412384":"2026-02-22T23:03:28.172242+0000","10.20.193.26:0/1347184781":"2026-02-22T23:00:57.133399+0000","10.20.193.26:0/3794703955":"2026-02-22T22:41:16.535269+0000","10.20.193.26:0/943080370":"2026-02-22T22:57:14.112717+0000","10.20.193.26:0/3347707812":"2026-02-22T23:02:26.201789+0000","10.20.193.26:0/2661804894":"2026-02-22T22:58:19.110581+0000","10.20.193.26:0/2938746720":"2026-02-22T22:58:49.096412+0000","10.20.193.26:0/1310228485":"2026-02-22T23:01:03.134752+0000","10.20.193.26:0/3584114150":"2026-02-22T23:03:22.151813+0000","10.20.193.26:0/989470677":"2026-02-22T22:57:58.502967+0000","10.20.193.26:0/1780094566":"2026-02-22T22:57:52.113569+0000","10.20.193.26:0/3776364600":"2026-02-22T22:57:27.999028+0000","10.20.193.26:0/825330076":"2026-02-22T22:44:40.469001+0000","10.20.193.26:0/3330509280":"2026-02-22T22:58:55.122255+0000","10.20.193.26:0/109867276":"2026-02-22T23:02:00.424121+0000","10.20.193.26:0/779693529":"2026-02-22T22:59:13.146773+0000","10.20.193.26:0/2923319013":"2026-02-22T22:45:58.098482+0000","10.20.193.26:0/610898380":"2026-02-22T23:08:02.892451+0000","10.20.193.26:0/1904235303":"2026-02-22T23:01:54.227754+0000","10.20.193.26:0/2237716999":"2026-02-22T22:39:34.938005+0000","10.20.193.26:0/143303415":"2026-02-22T23:01:36.136365+0000","10.20.193.26:0/3861462525":"2026-02-22T23:02:14.134613+0000","10.20.193.26:0/1203082057":"2026-02-22T22:40:28.896543+0000","10.20.193.26:0/3796738931":"2026-02-22T23:01:15.156592+0000","10.20.193.26:0/3510648365":"2026-02-22T22:42:10.099620+0000","10.20.193.26:0/1004385585":"2026-02-22T22:43:04.964292+0000","10.20.193.26:0/3722399939":"2026-02-22T22:59:01.128907+0000","10.20.193.26:0/3834404415":"2026-02-22T23:03:16.153441+0000","10.20.193.26:0/2665315594":"2026-02-22T22:57:20.360063+0000","10.20.193.26:0/407599080":"2026-02-22T22:58:12.283798+0000","10.20.193.26:0/336782528":"2026-02-22T22:57:40.103564+0000","10.20.193.26:0/3095225272":"2026-02-22T22:46:50.649988+0000","10.20.193.26:0/2181680172":"2026-02-22T23:02:20.113998+0000","10.20.193.26:0/37614622":"2026-02-22T22:57:01.394915+0000","10.20.193.26:0/1126556295":"2026-02-22T23:11:23.016329+0000","10.20.193.26:0/3783511748":"2026-02-22T22:59:19.148112+0000","10.20.193.26:0/1000419142":"2026-02-22T23:00:02.332411+0000","10.20.193.26:0/801638046":"2026-02-22T22:58:06.137650+0000","10.20.193.26:0/3537208109":"2026-02-22T23:01:42.184197+0000","10.20.193.26:0/1466750981":"2026-02-22T22:59:37.172559+0000","10.20.193.26:0/1664230592":"2026-02-22T23:01:09.165197+0000","10.20.193.26:0/1680462026":"2026-02-22T22:59:07.147710+0000","10.20.193.26:0/907070089":"2026-02-22T22:57:34.095432+0000","10.20.193.26:0/2007330267":"2026-02-22T22:57:08.112500+0000","10.20.193.26:0/353450893":"2026-02-22T22:59:25.162296+0000","10.20.193.26:0/3794234317":"2026-02-22T23:01:48.203617+0000","10.20.193.26:0/2465750359":"2026-02-22T22:59:31.168532+0000","10.20.193.26:0/1814272641":"2026-02-22T22:59:44.128095+0000","10.20.193.26:0/3764449858":"2026-02-22T22:38:43.985125+0000","10.20.193.26:0/100002871":"2026-02-22T22:59:50.139715+0000","10.20.193.26:0/3368556499":"2026-02-22T22:59:56.123806+0000","10.20.193.26:0/2759729631":"2026-02-22T23:00:08.335527+0000","10.20.193.26:0/538862641":"2026-02-22T23:00:14.349957+0000","10.20.193.26:0/746197305":"2026-02-22T23:00:50.339802+0000","10.20.193.26:0/1072275335":"2026-02-22T22:57:46.105454+0000","10.20.193.26:0/674081765":"2026-02-22T23:01:23.428506+0000","10.20.193.26:0/1624194683":"2026-02-22T23:00:44.127454+0000","10.20.193.26:0/670913725":"2026-02-22T23:01:30.140907+0000","10.20.193.26:0/1190964666":"2026-02-22T22:43:53.062426+0000","10.20.193.26:0/3554129092":"2026-02-22T23:02:32.196144+0000","10.20.193.26:0/1366729683":"2026-02-22T23:02:08.049496+0000","10.20.193.26:0/33905146":"2026-02-22T23:03:10.139234+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-22T22:12:11.170 INFO:tasks.ceph:Scrubbing cluster2.osd.0 2026-02-22T22:12:11.170 DEBUG:teuthology.orchestra.run.trial026:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.0 config set osd_debug_deep_scrub_sleep 0 2026-02-22T22:12:11.255 INFO:teuthology.orchestra.run.trial026.stdout:{ 2026-02-22T22:12:11.255 INFO:teuthology.orchestra.run.trial026.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) " 2026-02-22T22:12:11.255 INFO:teuthology.orchestra.run.trial026.stdout:} 2026-02-22T22:12:11.262 DEBUG:teuthology.orchestra.run.trial026:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd deep-scrub 0 2026-02-22T22:12:11.422 INFO:teuthology.orchestra.run.trial026.stderr:instructed osd(s) 0 to deep-scrub 2026-02-22T22:12:11.430 INFO:tasks.ceph:Scrubbing cluster2.osd.1 2026-02-22T22:12:11.430 DEBUG:teuthology.orchestra.run.trial026:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.1 config set osd_debug_deep_scrub_sleep 0 2026-02-22T22:12:11.510 INFO:teuthology.orchestra.run.trial026.stdout:{ 2026-02-22T22:12:11.510 INFO:teuthology.orchestra.run.trial026.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) " 2026-02-22T22:12:11.510 INFO:teuthology.orchestra.run.trial026.stdout:} 2026-02-22T22:12:11.516 DEBUG:teuthology.orchestra.run.trial026:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd deep-scrub 1 2026-02-22T22:12:11.672 INFO:teuthology.orchestra.run.trial026.stderr:instructed osd(s) 1 to deep-scrub 2026-02-22T22:12:11.681 INFO:tasks.ceph:Scrubbing cluster2.osd.2 2026-02-22T22:12:11.681 DEBUG:teuthology.orchestra.run.trial026:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.2 config set osd_debug_deep_scrub_sleep 0 2026-02-22T22:12:11.763 INFO:teuthology.orchestra.run.trial026.stdout:{ 2026-02-22T22:12:11.763 INFO:teuthology.orchestra.run.trial026.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) " 2026-02-22T22:12:11.763 INFO:teuthology.orchestra.run.trial026.stdout:} 2026-02-22T22:12:11.769 DEBUG:teuthology.orchestra.run.trial026:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd deep-scrub 2 2026-02-22T22:12:11.927 INFO:teuthology.orchestra.run.trial026.stderr:instructed osd(s) 2 to deep-scrub 2026-02-22T22:12:11.936 DEBUG:teuthology.orchestra.run.trial026:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2026-02-22T22:12:12.094 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T22:12:12.094 INFO:teuthology.orchestra.run.trial026.stderr:dumped all 2026-02-22T22:12:12.102 INFO:teuthology.orchestra.run.trial026.stdout:{"pg_ready":true,"pg_map":{"version":1514,"stamp":"2026-02-22T22:12:11.226102+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":59,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":2906892,"kb_used_data":15888,"kb_used_omap":297,"kb_used_meta":2890646,"kb_avail":280208628,"statfs":{"total":289910292480,"available":286933635072,"internally_reserved":0,"allocated":16269312,"data_stored":9094327,"data_compressed":19632,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":304727,"internal_metadata":2960021929},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-19,"num_objects":-17,"num_object_clones":0,"num_object_copies":-34,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-17,"num_whiteouts":0,"num_read":-2904,"num_read_kb":-2696,"num_write":-515,"num_write_kb":-332,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":-6,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.981516"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":907,"reported_epoch":454,"state":"active+clean","last_fresh":"2026-02-22T22:12:02.255700+0000","last_change":"2026-02-22T21:36:57.117998+0000","last_active":"2026-02-22T22:12:02.255700+0000","last_peered":"2026-02-22T22:12:02.255700+0000","last_clean":"2026-02-22T22:12:02.255700+0000","last_became_active":"2026-02-22T21:36:55.116298+0000","last_became_peered":"2026-02-22T21:36:55.116298+0000","last_unstale":"2026-02-22T22:12:02.255700+0000","last_undegraded":"2026-02-22T22:12:02.255700+0000","last_fullsized":"2026-02-22T22:12:02.255700+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:54.106037+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:54.106037+0000","last_clean_scrub_stamp":"2026-02-22T21:36:54.106037+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T02:34:53.292431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00037892999999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":907,"reported_epoch":454,"state":"active+clean","last_fresh":"2026-02-22T22:12:02.258275+0000","last_change":"2026-02-22T21:36:57.118108+0000","last_active":"2026-02-22T22:12:02.258275+0000","last_peered":"2026-02-22T22:12:02.258275+0000","last_clean":"2026-02-22T22:12:02.258275+0000","last_became_active":"2026-02-22T21:36:55.116070+0000","last_became_peered":"2026-02-22T21:36:55.116070+0000","last_unstale":"2026-02-22T22:12:02.258275+0000","last_undegraded":"2026-02-22T22:12:02.258275+0000","last_fullsized":"2026-02-22T22:12:02.258275+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:54.106037+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:54.106037+0000","last_clean_scrub_stamp":"2026-02-22T21:36:54.106037+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T07:44:09.720355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048051700000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":907,"reported_epoch":454,"state":"active+clean","last_fresh":"2026-02-22T22:12:02.259437+0000","last_change":"2026-02-22T21:36:57.118060+0000","last_active":"2026-02-22T22:12:02.259437+0000","last_peered":"2026-02-22T22:12:02.259437+0000","last_clean":"2026-02-22T22:12:02.259437+0000","last_became_active":"2026-02-22T21:36:55.116352+0000","last_became_peered":"2026-02-22T21:36:55.116352+0000","last_unstale":"2026-02-22T22:12:02.259437+0000","last_undegraded":"2026-02-22T22:12:02.259437+0000","last_fullsized":"2026-02-22T22:12:02.259437+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:54.106037+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:54.106037+0000","last_clean_scrub_stamp":"2026-02-22T21:36:54.106037+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T00:50:23.097350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00044389899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":907,"reported_epoch":454,"state":"active+clean","last_fresh":"2026-02-22T22:12:02.253765+0000","last_change":"2026-02-22T21:36:57.117889+0000","last_active":"2026-02-22T22:12:02.253765+0000","last_peered":"2026-02-22T22:12:02.253765+0000","last_clean":"2026-02-22T22:12:02.253765+0000","last_became_active":"2026-02-22T21:36:55.116352+0000","last_became_peered":"2026-02-22T21:36:55.116352+0000","last_unstale":"2026-02-22T22:12:02.253765+0000","last_undegraded":"2026-02-22T22:12:02.253765+0000","last_fullsized":"2026-02-22T22:12:02.253765+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:54.106037+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:54.106037+0000","last_clean_scrub_stamp":"2026-02-22T21:36:54.106037+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T06:10:01.562314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029249000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"16'2","reported_seq":912,"reported_epoch":454,"state":"active+clean","last_fresh":"2026-02-22T22:12:03.213071+0000","last_change":"2026-02-22T21:36:57.116752+0000","last_active":"2026-02-22T22:12:03.213071+0000","last_peered":"2026-02-22T22:12:03.213071+0000","last_clean":"2026-02-22T22:12:03.213071+0000","last_became_active":"2026-02-22T21:36:55.116275+0000","last_became_peered":"2026-02-22T21:36:55.116275+0000","last_unstale":"2026-02-22T22:12:03.213071+0000","last_undegraded":"2026-02-22T22:12:03.213071+0000","last_fullsized":"2026-02-22T22:12:03.213071+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:54.106037+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:54.106037+0000","last_clean_scrub_stamp":"2026-02-22T21:36:54.106037+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T07:41:58.739291+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000424664,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":908,"reported_epoch":454,"state":"active+clean","last_fresh":"2026-02-22T22:12:03.251863+0000","last_change":"2026-02-22T21:36:57.115505+0000","last_active":"2026-02-22T22:12:03.251863+0000","last_peered":"2026-02-22T22:12:03.251863+0000","last_clean":"2026-02-22T22:12:03.251863+0000","last_became_active":"2026-02-22T21:36:55.116210+0000","last_became_peered":"2026-02-22T21:36:55.116210+0000","last_unstale":"2026-02-22T22:12:03.251863+0000","last_undegraded":"2026-02-22T22:12:03.251863+0000","last_fullsized":"2026-02-22T22:12:03.251863+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:54.106037+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:54.106037+0000","last_clean_scrub_stamp":"2026-02-22T21:36:54.106037+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T02:21:43.966839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024309900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":909,"reported_epoch":454,"state":"active+clean","last_fresh":"2026-02-22T22:12:03.251313+0000","last_change":"2026-02-22T21:36:57.115504+0000","last_active":"2026-02-22T22:12:03.251313+0000","last_peered":"2026-02-22T22:12:03.251313+0000","last_clean":"2026-02-22T22:12:03.251313+0000","last_became_active":"2026-02-22T21:36:55.116216+0000","last_became_peered":"2026-02-22T21:36:55.116216+0000","last_unstale":"2026-02-22T22:12:03.251313+0000","last_undegraded":"2026-02-22T22:12:03.251313+0000","last_fullsized":"2026-02-22T22:12:03.251313+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:54.106037+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:54.106037+0000","last_clean_scrub_stamp":"2026-02-22T21:36:54.106037+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T23:54:52.644119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000245073,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"14'1","reported_seq":908,"reported_epoch":454,"state":"active+clean","last_fresh":"2026-02-22T22:12:02.257778+0000","last_change":"2026-02-22T21:36:57.117874+0000","last_active":"2026-02-22T22:12:02.257778+0000","last_peered":"2026-02-22T22:12:02.257778+0000","last_clean":"2026-02-22T22:12:02.257778+0000","last_became_active":"2026-02-22T21:36:55.116006+0000","last_became_peered":"2026-02-22T21:36:55.116006+0000","last_unstale":"2026-02-22T22:12:02.257778+0000","last_undegraded":"2026-02-22T22:12:02.257778+0000","last_fullsized":"2026-02-22T22:12:02.257778+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:54.106037+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:54.106037+0000","last_clean_scrub_stamp":"2026-02-22T21:36:54.106037+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T03:58:14.400632+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029477300000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"11'72","reported_seq":999,"reported_epoch":454,"state":"active+clean","last_fresh":"2026-02-22T22:12:02.252737+0000","last_change":"2026-02-22T21:36:50.308369+0000","last_active":"2026-02-22T22:12:02.252737+0000","last_peered":"2026-02-22T22:12:02.252737+0000","last_clean":"2026-02-22T22:12:02.252737+0000","last_became_active":"2026-02-22T21:36:50.308099+0000","last_became_peered":"2026-02-22T21:36:50.308099+0000","last_unstale":"2026-02-22T22:12:02.252737+0000","last_undegraded":"2026-02-22T22:12:02.252737+0000","last_fullsized":"2026-02-22T22:12:02.252737+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:49.099350+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:49.099350+0000","last_clean_scrub_stamp":"2026-02-22T21:36:49.099350+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T09:35:05.112768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":1180736,"data_compressed":19632,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":1,"up_from":9,"seq":38654706092,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":945048,"kb_used_data":5316,"kb_used_omap":76,"kb_used_meta":939635,"kb_avail":93426792,"statfs":{"total":96636764160,"available":95669035008,"internally_reserved":0,"allocated":5443584,"data_stored":3228238,"data_compressed":9816,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":78140,"internal_metadata":962186948},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":9,"seq":38654706092,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1011032,"kb_used_data":5316,"kb_used_omap":126,"kb_used_meta":1005569,"kb_avail":93360808,"statfs":{"total":96636764160,"available":95601467392,"internally_reserved":0,"allocated":5443584,"data_stored":3228238,"data_compressed":9816,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":129039,"internal_metadata":1029703665},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":8,"seq":34359738796,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":950812,"kb_used_data":5256,"kb_used_omap":95,"kb_used_meta":945440,"kb_avail":93421028,"statfs":{"total":96636764160,"available":95663132672,"internally_reserved":0,"allocated":5382144,"data_stored":2637851,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":97548,"internal_metadata":968131316},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":9816,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":9816,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-22T22:12:12.107 INFO:tasks.ceph:pgid 2.7 last_scrub_stamp 2026-02-22T21:36:54.106037+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=21, tm_min=36, tm_sec=54, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=12, tm_sec=10, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T22:12:12.108 INFO:tasks.ceph:pgid 2.6 last_scrub_stamp 2026-02-22T21:36:54.106037+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=21, tm_min=36, tm_sec=54, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=12, tm_sec=10, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T22:12:12.108 INFO:tasks.ceph:pgid 2.5 last_scrub_stamp 2026-02-22T21:36:54.106037+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=21, tm_min=36, tm_sec=54, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=12, tm_sec=10, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T22:12:12.108 INFO:tasks.ceph:pgid 2.4 last_scrub_stamp 2026-02-22T21:36:54.106037+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=21, tm_min=36, tm_sec=54, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=12, tm_sec=10, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T22:12:12.108 INFO:tasks.ceph:pgid 2.2 last_scrub_stamp 2026-02-22T21:36:54.106037+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=21, tm_min=36, tm_sec=54, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=12, tm_sec=10, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T22:12:12.108 INFO:tasks.ceph:pgid 2.1 last_scrub_stamp 2026-02-22T21:36:54.106037+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=21, tm_min=36, tm_sec=54, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=12, tm_sec=10, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T22:12:12.109 INFO:tasks.ceph:pgid 2.0 last_scrub_stamp 2026-02-22T21:36:54.106037+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=21, tm_min=36, tm_sec=54, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=12, tm_sec=10, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T22:12:12.109 INFO:tasks.ceph:pgid 2.3 last_scrub_stamp 2026-02-22T21:36:54.106037+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=21, tm_min=36, tm_sec=54, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=12, tm_sec=10, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T22:12:12.109 INFO:tasks.ceph:pgid 1.0 last_scrub_stamp 2026-02-22T21:36:49.099350+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=21, tm_min=36, tm_sec=49, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=12, tm_sec=10, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T22:12:12.109 INFO:tasks.ceph:Still waiting for all pgs to be scrubbed. 2026-02-22T22:12:32.109 DEBUG:teuthology.orchestra.run.trial026:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2026-02-22T22:12:32.275 INFO:teuthology.orchestra.run.trial026.stdout: 2026-02-22T22:12:32.276 INFO:teuthology.orchestra.run.trial026.stderr:dumped all 2026-02-22T22:12:32.284 INFO:teuthology.orchestra.run.trial026.stdout:{"pg_ready":true,"pg_map":{"version":1524,"stamp":"2026-02-22T22:12:31.227506+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":79,"ondisk_log_size":79,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":2906880,"kb_used_data":15888,"kb_used_omap":297,"kb_used_meta":2890646,"kb_avail":280208640,"statfs":{"total":289910292480,"available":286933647360,"internally_reserved":0,"allocated":16269312,"data_stored":9094327,"data_compressed":19632,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":304727,"internal_metadata":2960021929},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.000820"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":916,"reported_epoch":454,"state":"active+clean","last_fresh":"2026-02-22T22:12:14.565629+0000","last_change":"2026-02-22T22:12:14.565640+0000","last_active":"2026-02-22T22:12:14.565629+0000","last_peered":"2026-02-22T22:12:14.565629+0000","last_clean":"2026-02-22T22:12:14.565629+0000","last_became_active":"2026-02-22T21:36:55.116298+0000","last_became_peered":"2026-02-22T21:36:55.116298+0000","last_unstale":"2026-02-22T22:12:14.565629+0000","last_undegraded":"2026-02-22T22:12:14.565629+0000","last_fullsized":"2026-02-22T22:12:14.565629+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:12:14.565581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:12:14.565581+0000","last_clean_scrub_stamp":"2026-02-22T22:12:14.565581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T07:47:35.203381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00037892999999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":916,"reported_epoch":454,"state":"active+clean","last_fresh":"2026-02-22T22:12:13.575034+0000","last_change":"2026-02-22T22:12:13.575046+0000","last_active":"2026-02-22T22:12:13.575034+0000","last_peered":"2026-02-22T22:12:13.575034+0000","last_clean":"2026-02-22T22:12:13.575034+0000","last_became_active":"2026-02-22T21:36:55.116070+0000","last_became_peered":"2026-02-22T21:36:55.116070+0000","last_unstale":"2026-02-22T22:12:13.575034+0000","last_undegraded":"2026-02-22T22:12:13.575034+0000","last_fullsized":"2026-02-22T22:12:13.575034+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:12:13.574753+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:12:13.574753+0000","last_clean_scrub_stamp":"2026-02-22T22:12:13.574753+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T07:16:34.989202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048051700000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":917,"reported_epoch":454,"state":"active+clean","last_fresh":"2026-02-22T22:12:16.552268+0000","last_change":"2026-02-22T22:12:16.552268+0000","last_active":"2026-02-22T22:12:16.552268+0000","last_peered":"2026-02-22T22:12:16.552268+0000","last_clean":"2026-02-22T22:12:16.552268+0000","last_became_active":"2026-02-22T21:36:55.116352+0000","last_became_peered":"2026-02-22T21:36:55.116352+0000","last_unstale":"2026-02-22T22:12:16.552268+0000","last_undegraded":"2026-02-22T22:12:16.552268+0000","last_fullsized":"2026-02-22T22:12:16.552268+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:12:16.552206+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:12:16.552206+0000","last_clean_scrub_stamp":"2026-02-22T22:12:16.552206+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:25:02.072003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00044389899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":917,"reported_epoch":454,"state":"active+clean","last_fresh":"2026-02-22T22:12:15.601518+0000","last_change":"2026-02-22T22:12:15.601518+0000","last_active":"2026-02-22T22:12:15.601518+0000","last_peered":"2026-02-22T22:12:15.601518+0000","last_clean":"2026-02-22T22:12:15.601518+0000","last_became_active":"2026-02-22T21:36:55.116352+0000","last_became_peered":"2026-02-22T21:36:55.116352+0000","last_unstale":"2026-02-22T22:12:15.601518+0000","last_undegraded":"2026-02-22T22:12:15.601518+0000","last_fullsized":"2026-02-22T22:12:15.601518+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:12:15.601453+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:12:15.601453+0000","last_clean_scrub_stamp":"2026-02-22T22:12:15.601453+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T00:01:12.025829+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029249000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"454'3","reported_seq":923,"reported_epoch":454,"state":"active+clean","last_fresh":"2026-02-22T22:12:12.223986+0000","last_change":"2026-02-22T22:12:12.223986+0000","last_active":"2026-02-22T22:12:12.223986+0000","last_peered":"2026-02-22T22:12:12.223986+0000","last_clean":"2026-02-22T22:12:12.223986+0000","last_became_active":"2026-02-22T21:36:55.116275+0000","last_became_peered":"2026-02-22T21:36:55.116275+0000","last_unstale":"2026-02-22T22:12:12.223986+0000","last_undegraded":"2026-02-22T22:12:12.223986+0000","last_fullsized":"2026-02-22T22:12:12.223986+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"454'3","last_scrub_stamp":"2026-02-22T22:12:12.223884+0000","last_deep_scrub":"454'3","last_deep_scrub_stamp":"2026-02-22T22:12:12.223884+0000","last_clean_scrub_stamp":"2026-02-22T22:12:12.223884+0000","objects_scrubbed":1,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T10:08:02.302448+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0.000424664,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":918,"reported_epoch":454,"state":"active+clean","last_fresh":"2026-02-22T22:12:13.333813+0000","last_change":"2026-02-22T22:12:13.333813+0000","last_active":"2026-02-22T22:12:13.333813+0000","last_peered":"2026-02-22T22:12:13.333813+0000","last_clean":"2026-02-22T22:12:13.333813+0000","last_became_active":"2026-02-22T21:36:55.116210+0000","last_became_peered":"2026-02-22T21:36:55.116210+0000","last_unstale":"2026-02-22T22:12:13.333813+0000","last_undegraded":"2026-02-22T22:12:13.333813+0000","last_fullsized":"2026-02-22T22:12:13.333813+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:12:13.333741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:12:13.333741+0000","last_clean_scrub_stamp":"2026-02-22T22:12:13.333741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T22:49:03.184882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024309900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":918,"reported_epoch":454,"state":"active+clean","last_fresh":"2026-02-22T22:12:12.371779+0000","last_change":"2026-02-22T22:12:12.371802+0000","last_active":"2026-02-22T22:12:12.371779+0000","last_peered":"2026-02-22T22:12:12.371779+0000","last_clean":"2026-02-22T22:12:12.371779+0000","last_became_active":"2026-02-22T21:36:55.116216+0000","last_became_peered":"2026-02-22T21:36:55.116216+0000","last_unstale":"2026-02-22T22:12:12.371779+0000","last_undegraded":"2026-02-22T22:12:12.371779+0000","last_fullsized":"2026-02-22T22:12:12.371779+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:12:12.371716+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:12:12.371716+0000","last_clean_scrub_stamp":"2026-02-22T22:12:12.371716+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T09:31:50.330456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000245073,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"454'2","reported_seq":919,"reported_epoch":454,"state":"active+clean","last_fresh":"2026-02-22T22:12:17.602089+0000","last_change":"2026-02-22T22:12:17.602089+0000","last_active":"2026-02-22T22:12:17.602089+0000","last_peered":"2026-02-22T22:12:17.602089+0000","last_clean":"2026-02-22T22:12:17.602089+0000","last_became_active":"2026-02-22T21:36:55.116006+0000","last_became_peered":"2026-02-22T21:36:55.116006+0000","last_unstale":"2026-02-22T22:12:17.602089+0000","last_undegraded":"2026-02-22T22:12:17.602089+0000","last_fullsized":"2026-02-22T22:12:17.602089+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"454'2","last_scrub_stamp":"2026-02-22T22:12:17.602002+0000","last_deep_scrub":"454'2","last_deep_scrub_stamp":"2026-02-22T22:12:17.602002+0000","last_clean_scrub_stamp":"2026-02-22T22:12:17.602002+0000","objects_scrubbed":1,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T05:32:29.573822+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0.00029477300000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"454'74","reported_seq":1010,"reported_epoch":454,"state":"active+clean","last_fresh":"2026-02-22T22:12:12.536735+0000","last_change":"2026-02-22T22:12:12.536748+0000","last_active":"2026-02-22T22:12:12.536735+0000","last_peered":"2026-02-22T22:12:12.536735+0000","last_clean":"2026-02-22T22:12:12.536735+0000","last_became_active":"2026-02-22T21:36:50.308099+0000","last_became_peered":"2026-02-22T21:36:50.308099+0000","last_unstale":"2026-02-22T22:12:12.536735+0000","last_undegraded":"2026-02-22T22:12:12.536735+0000","last_fullsized":"2026-02-22T22:12:12.536735+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"454'74","last_scrub_stamp":"2026-02-22T22:12:12.536696+0000","last_deep_scrub":"454'74","last_deep_scrub_stamp":"2026-02-22T22:12:12.536696+0000","last_clean_scrub_stamp":"2026-02-22T22:12:12.536696+0000","objects_scrubbed":2,"log_size":74,"log_dups_size":0,"ondisk_log_size":74,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T23:07:03.053028+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":5,"ondisk_log_size":5,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":1180736,"data_compressed":19632,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":0,"internal_metadata":0},"log_size":74,"ondisk_log_size":74,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":1,"up_from":9,"seq":38654706096,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":945044,"kb_used_data":5316,"kb_used_omap":76,"kb_used_meta":939635,"kb_avail":93426796,"statfs":{"total":96636764160,"available":95669039104,"internally_reserved":0,"allocated":5443584,"data_stored":3228238,"data_compressed":9816,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":78140,"internal_metadata":962186948},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":9,"seq":38654706096,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1011028,"kb_used_data":5316,"kb_used_omap":126,"kb_used_meta":1005569,"kb_avail":93360812,"statfs":{"total":96636764160,"available":95601471488,"internally_reserved":0,"allocated":5443584,"data_stored":3228238,"data_compressed":9816,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":129039,"internal_metadata":1029703665},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":8,"seq":34359738800,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":950808,"kb_used_data":5256,"kb_used_omap":95,"kb_used_meta":945440,"kb_avail":93421032,"statfs":{"total":96636764160,"available":95663136768,"internally_reserved":0,"allocated":5382144,"data_stored":2637851,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":97548,"internal_metadata":968131316},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":9816,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":9816,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-22T22:12:32.286 DEBUG:teuthology.orchestra.run.trial026:> sudo ceph --cluster cluster2 config set global mon_health_to_clog false 2026-02-22T22:12:32.455 INFO:teuthology.misc:Shutting down mds daemons... 2026-02-22T22:12:32.456 INFO:teuthology.misc:Shutting down osd daemons... 2026-02-22T22:12:32.456 DEBUG:tasks.ceph.cluster2.osd.0:waiting for process to exit 2026-02-22T22:12:32.456 INFO:teuthology.orchestra.run:waiting for 300 2026-02-22T22:12:32.515 INFO:tasks.ceph.cluster2.osd.0:Stopped 2026-02-22T22:12:32.515 DEBUG:tasks.ceph.cluster2.osd.1:waiting for process to exit 2026-02-22T22:12:32.515 INFO:teuthology.orchestra.run:waiting for 300 2026-02-22T22:12:32.592 INFO:tasks.ceph.cluster2.osd.1:Stopped 2026-02-22T22:12:32.592 DEBUG:tasks.ceph.cluster2.osd.2:waiting for process to exit 2026-02-22T22:12:32.592 INFO:teuthology.orchestra.run:waiting for 300 2026-02-22T22:12:32.663 INFO:tasks.ceph.cluster2.osd.2:Stopped 2026-02-22T22:12:32.664 INFO:teuthology.misc:Shutting down mgr daemons... 2026-02-22T22:12:32.664 DEBUG:tasks.ceph.cluster2.mgr.x:waiting for process to exit 2026-02-22T22:12:32.664 INFO:teuthology.orchestra.run:waiting for 300 2026-02-22T22:12:32.706 INFO:tasks.ceph.cluster2.mgr.x:Stopped 2026-02-22T22:12:32.707 INFO:teuthology.misc:Shutting down mon daemons... 2026-02-22T22:12:32.707 DEBUG:tasks.ceph.cluster2.mon.a:waiting for process to exit 2026-02-22T22:12:32.707 INFO:teuthology.orchestra.run:waiting for 300 2026-02-22T22:12:32.720 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.2 is failed for ~0s 2026-02-22T22:12:32.720 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.mon.a is failed for ~0s 2026-02-22T22:12:32.720 INFO:tasks.ceph.cluster2.mon.a:Stopped 2026-02-22T22:12:32.720 INFO:tasks.ceph:Checking cluster log for badness... 2026-02-22T22:12:32.721 DEBUG:teuthology.orchestra.run.trial026:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/cluster2.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v '\(OSD_SLOW_PING_TIME' | head -n 1 2026-02-22T22:12:32.786 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster2-0 on ubuntu@trial026.front.sepia.ceph.com 2026-02-22T22:12:32.787 DEBUG:teuthology.orchestra.run.trial026:> sync && sudo umount -f /var/lib/ceph/osd/cluster2-0 2026-02-22T22:12:33.023 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster2-1 on ubuntu@trial026.front.sepia.ceph.com 2026-02-22T22:12:33.024 DEBUG:teuthology.orchestra.run.trial026:> sync && sudo umount -f /var/lib/ceph/osd/cluster2-1 2026-02-22T22:12:33.216 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster2-2 on ubuntu@trial026.front.sepia.ceph.com 2026-02-22T22:12:33.216 DEBUG:teuthology.orchestra.run.trial026:> sync && sudo umount -f /var/lib/ceph/osd/cluster2-2 2026-02-22T22:12:33.402 INFO:tasks.ceph:Archiving mon data... 2026-02-22T22:12:33.408 DEBUG:teuthology.misc:Transferring archived files from trial026:/var/lib/ceph/mon/cluster2-a to /home/teuthworker/mnt/teuthology/yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial/64060/data/cluster2.mon.a.tgz 2026-02-22T22:12:33.408 DEBUG:teuthology.orchestra.run.trial026:> mktemp 2026-02-22T22:12:33.425 INFO:teuthology.orchestra.run.trial026.stdout:/tmp/tmp.Mp8aWNr3EJ 2026-02-22T22:12:33.425 DEBUG:teuthology.orchestra.run.trial026:> sudo tar cz -f - -C /var/lib/ceph/mon/cluster2-a -- . > /tmp/tmp.Mp8aWNr3EJ 2026-02-22T22:12:33.545 DEBUG:teuthology.orchestra.run.trial026:> sudo chmod 0666 /tmp/tmp.Mp8aWNr3EJ 2026-02-22T22:12:33.627 DEBUG:teuthology.orchestra.remote:trial026:/tmp/tmp.Mp8aWNr3EJ is 438KB 2026-02-22T22:12:33.724 DEBUG:teuthology.orchestra.run.trial026:> rm -fr /tmp/tmp.Mp8aWNr3EJ 2026-02-22T22:12:33.742 INFO:tasks.ceph:Cleaning ceph cluster... 2026-02-22T22:12:33.742 DEBUG:teuthology.orchestra.run.trial004:> sudo rm -rf -- /etc/ceph/cluster2.conf /etc/ceph/cluster2.keyring /home/ubuntu/cephtest/cluster2.data /home/ubuntu/cephtest/cluster2.monmap /home/ubuntu/cephtest/../*.pid 2026-02-22T22:12:33.744 DEBUG:teuthology.orchestra.run.trial026:> sudo rm -rf -- /etc/ceph/cluster2.conf /etc/ceph/cluster2.keyring /home/ubuntu/cephtest/cluster2.data /home/ubuntu/cephtest/cluster2.monmap /home/ubuntu/cephtest/../*.pid 2026-02-22T22:12:33.806 DEBUG:teuthology.run_tasks:Unwinding manager ceph 2026-02-22T22:12:33.827 INFO:tasks.ceph.ceph_manager.cluster1:waiting for clean 2026-02-22T22:12:33.827 DEBUG:teuthology.orchestra.run.trial004:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2026-02-22T22:12:33.984 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T22:12:33.985 INFO:teuthology.orchestra.run.trial004.stderr:dumped all 2026-02-22T22:12:33.992 INFO:teuthology.orchestra.run.trial004.stdout:{"pg_ready":true,"pg_map":{"version":1543,"stamp":"2026-02-22T22:12:32.934308+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":257084,"kb_used_data":11064,"kb_used_omap":4,"kb_used_meta":245947,"kb_avail":282858436,"statfs":{"total":289910292480,"available":289647038464,"internally_reserved":0,"allocated":11329536,"data_stored":7174951,"data_compressed":19658,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":4782,"internal_metadata":251850066},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.000847"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":909,"reported_epoch":454,"state":"active+clean","last_fresh":"2026-02-22T22:12:02.334605+0000","last_change":"2026-02-22T21:36:19.704598+0000","last_active":"2026-02-22T22:12:02.334605+0000","last_peered":"2026-02-22T22:12:02.334605+0000","last_clean":"2026-02-22T22:12:02.334605+0000","last_became_active":"2026-02-22T21:36:17.702763+0000","last_became_peered":"2026-02-22T21:36:17.702763+0000","last_unstale":"2026-02-22T22:12:02.334605+0000","last_undegraded":"2026-02-22T22:12:02.334605+0000","last_fullsized":"2026-02-22T22:12:02.334605+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:16.695655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:16.695655+0000","last_clean_scrub_stamp":"2026-02-22T21:36:16.695655+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T21:59:04.606522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00062915900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":909,"reported_epoch":454,"state":"active+clean","last_fresh":"2026-02-22T22:12:02.336125+0000","last_change":"2026-02-22T21:36:19.704496+0000","last_active":"2026-02-22T22:12:02.336125+0000","last_peered":"2026-02-22T22:12:02.336125+0000","last_clean":"2026-02-22T22:12:02.336125+0000","last_became_active":"2026-02-22T21:36:17.703106+0000","last_became_peered":"2026-02-22T21:36:17.703106+0000","last_unstale":"2026-02-22T22:12:02.336125+0000","last_undegraded":"2026-02-22T22:12:02.336125+0000","last_fullsized":"2026-02-22T22:12:02.336125+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:16.695655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:16.695655+0000","last_clean_scrub_stamp":"2026-02-22T21:36:16.695655+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T03:56:26.669952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00053011499999999995,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":910,"reported_epoch":454,"state":"active+clean","last_fresh":"2026-02-22T22:12:02.335481+0000","last_change":"2026-02-22T21:36:19.704255+0000","last_active":"2026-02-22T22:12:02.335481+0000","last_peered":"2026-02-22T22:12:02.335481+0000","last_clean":"2026-02-22T22:12:02.335481+0000","last_became_active":"2026-02-22T21:36:17.702758+0000","last_became_peered":"2026-02-22T21:36:17.702758+0000","last_unstale":"2026-02-22T22:12:02.335481+0000","last_undegraded":"2026-02-22T22:12:02.335481+0000","last_fullsized":"2026-02-22T22:12:02.335481+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:16.695655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:16.695655+0000","last_clean_scrub_stamp":"2026-02-22T21:36:16.695655+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T01:00:00.761061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033481199999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":910,"reported_epoch":454,"state":"active+clean","last_fresh":"2026-02-22T22:12:02.334876+0000","last_change":"2026-02-22T21:36:19.704343+0000","last_active":"2026-02-22T22:12:02.334876+0000","last_peered":"2026-02-22T22:12:02.334876+0000","last_clean":"2026-02-22T22:12:02.334876+0000","last_became_active":"2026-02-22T21:36:17.702784+0000","last_became_peered":"2026-02-22T21:36:17.702784+0000","last_unstale":"2026-02-22T22:12:02.334876+0000","last_undegraded":"2026-02-22T22:12:02.334876+0000","last_fullsized":"2026-02-22T22:12:02.334876+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:16.695655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:16.695655+0000","last_clean_scrub_stamp":"2026-02-22T21:36:16.695655+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T01:51:15.121856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000400474,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"16'2","reported_seq":912,"reported_epoch":454,"state":"active+clean","last_fresh":"2026-02-22T22:12:03.093192+0000","last_change":"2026-02-22T21:36:19.706373+0000","last_active":"2026-02-22T22:12:03.093192+0000","last_peered":"2026-02-22T22:12:03.093192+0000","last_clean":"2026-02-22T22:12:03.093192+0000","last_became_active":"2026-02-22T21:36:17.702836+0000","last_became_peered":"2026-02-22T21:36:17.702836+0000","last_unstale":"2026-02-22T22:12:03.093192+0000","last_undegraded":"2026-02-22T22:12:03.093192+0000","last_fullsized":"2026-02-22T22:12:03.093192+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:16.695655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:16.695655+0000","last_clean_scrub_stamp":"2026-02-22T21:36:16.695655+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T09:04:07.404872+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00089484299999999998,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":912,"reported_epoch":454,"state":"active+clean","last_fresh":"2026-02-22T22:12:01.506040+0000","last_change":"2026-02-22T21:36:19.749070+0000","last_active":"2026-02-22T22:12:01.506040+0000","last_peered":"2026-02-22T22:12:01.506040+0000","last_clean":"2026-02-22T22:12:01.506040+0000","last_became_active":"2026-02-22T21:36:17.702842+0000","last_became_peered":"2026-02-22T21:36:17.702842+0000","last_unstale":"2026-02-22T22:12:01.506040+0000","last_undegraded":"2026-02-22T22:12:01.506040+0000","last_fullsized":"2026-02-22T22:12:01.506040+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:16.695655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:16.695655+0000","last_clean_scrub_stamp":"2026-02-22T21:36:16.695655+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T08:47:34.298671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00023738200000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":913,"reported_epoch":454,"state":"active+clean","last_fresh":"2026-02-22T22:12:01.504892+0000","last_change":"2026-02-22T21:36:19.749053+0000","last_active":"2026-02-22T22:12:01.504892+0000","last_peered":"2026-02-22T22:12:01.504892+0000","last_clean":"2026-02-22T22:12:01.504892+0000","last_became_active":"2026-02-22T21:36:17.702618+0000","last_became_peered":"2026-02-22T21:36:17.702618+0000","last_unstale":"2026-02-22T22:12:01.504892+0000","last_undegraded":"2026-02-22T22:12:01.504892+0000","last_fullsized":"2026-02-22T22:12:01.504892+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:16.695655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:16.695655+0000","last_clean_scrub_stamp":"2026-02-22T21:36:16.695655+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T23:00:17.618828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026033399999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"14'1","reported_seq":911,"reported_epoch":454,"state":"active+clean","last_fresh":"2026-02-22T22:12:02.331103+0000","last_change":"2026-02-22T21:36:19.704451+0000","last_active":"2026-02-22T22:12:02.331103+0000","last_peered":"2026-02-22T22:12:02.331103+0000","last_clean":"2026-02-22T22:12:02.331103+0000","last_became_active":"2026-02-22T21:36:17.702257+0000","last_became_peered":"2026-02-22T21:36:17.702257+0000","last_unstale":"2026-02-22T22:12:02.331103+0000","last_undegraded":"2026-02-22T22:12:02.331103+0000","last_fullsized":"2026-02-22T22:12:02.331103+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:16.695655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:16.695655+0000","last_clean_scrub_stamp":"2026-02-22T21:36:16.695655+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T01:42:15.152043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00049669299999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"11'72","reported_seq":1002,"reported_epoch":454,"state":"active+clean","last_fresh":"2026-02-22T22:12:02.334763+0000","last_change":"2026-02-22T21:36:14.450229+0000","last_active":"2026-02-22T22:12:02.334763+0000","last_peered":"2026-02-22T22:12:02.334763+0000","last_clean":"2026-02-22T22:12:02.334763+0000","last_became_active":"2026-02-22T21:36:14.449500+0000","last_became_peered":"2026-02-22T21:36:14.449500+0000","last_unstale":"2026-02-22T22:12:02.334763+0000","last_undegraded":"2026-02-22T22:12:02.334763+0000","last_fullsized":"2026-02-22T22:12:02.334763+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:12.692166+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:12.692166+0000","last_clean_scrub_stamp":"2026-02-22T21:36:12.692166+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T02:36:42.881463+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":1180736,"data_compressed":19658,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738808,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":75032,"kb_used_data":3648,"kb_used_omap":1,"kb_used_meta":71358,"kb_avail":94296808,"statfs":{"total":96636764160,"available":96559931392,"internally_reserved":0,"allocated":3735552,"data_stored":1998059,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":73071046},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738809,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":70928,"kb_used_data":3708,"kb_used_omap":1,"kb_used_meta":67198,"kb_avail":94300912,"statfs":{"total":96636764160,"available":96564133888,"internally_reserved":0,"allocated":3796992,"data_stored":2588446,"data_compressed":9829,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1594,"internal_metadata":68811206},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738809,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":111124,"kb_used_data":3708,"kb_used_omap":1,"kb_used_meta":107390,"kb_avail":94260716,"statfs":{"total":96636764160,"available":96522973184,"internally_reserved":0,"allocated":3796992,"data_stored":2588446,"data_compressed":9829,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1594,"internal_metadata":109967814},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":9829,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":9829,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-22T22:12:33.993 DEBUG:teuthology.orchestra.run.trial004:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2026-02-22T22:12:34.150 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T22:12:34.150 INFO:teuthology.orchestra.run.trial004.stderr:dumped all 2026-02-22T22:12:34.158 INFO:teuthology.orchestra.run.trial004.stdout:{"pg_ready":true,"pg_map":{"version":1543,"stamp":"2026-02-22T22:12:32.934308+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":257084,"kb_used_data":11064,"kb_used_omap":4,"kb_used_meta":245947,"kb_avail":282858436,"statfs":{"total":289910292480,"available":289647038464,"internally_reserved":0,"allocated":11329536,"data_stored":7174951,"data_compressed":19658,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":4782,"internal_metadata":251850066},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.000847"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":909,"reported_epoch":454,"state":"active+clean","last_fresh":"2026-02-22T22:12:02.334605+0000","last_change":"2026-02-22T21:36:19.704598+0000","last_active":"2026-02-22T22:12:02.334605+0000","last_peered":"2026-02-22T22:12:02.334605+0000","last_clean":"2026-02-22T22:12:02.334605+0000","last_became_active":"2026-02-22T21:36:17.702763+0000","last_became_peered":"2026-02-22T21:36:17.702763+0000","last_unstale":"2026-02-22T22:12:02.334605+0000","last_undegraded":"2026-02-22T22:12:02.334605+0000","last_fullsized":"2026-02-22T22:12:02.334605+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:16.695655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:16.695655+0000","last_clean_scrub_stamp":"2026-02-22T21:36:16.695655+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T21:59:04.606522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00062915900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":909,"reported_epoch":454,"state":"active+clean","last_fresh":"2026-02-22T22:12:02.336125+0000","last_change":"2026-02-22T21:36:19.704496+0000","last_active":"2026-02-22T22:12:02.336125+0000","last_peered":"2026-02-22T22:12:02.336125+0000","last_clean":"2026-02-22T22:12:02.336125+0000","last_became_active":"2026-02-22T21:36:17.703106+0000","last_became_peered":"2026-02-22T21:36:17.703106+0000","last_unstale":"2026-02-22T22:12:02.336125+0000","last_undegraded":"2026-02-22T22:12:02.336125+0000","last_fullsized":"2026-02-22T22:12:02.336125+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:16.695655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:16.695655+0000","last_clean_scrub_stamp":"2026-02-22T21:36:16.695655+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T03:56:26.669952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00053011499999999995,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":910,"reported_epoch":454,"state":"active+clean","last_fresh":"2026-02-22T22:12:02.335481+0000","last_change":"2026-02-22T21:36:19.704255+0000","last_active":"2026-02-22T22:12:02.335481+0000","last_peered":"2026-02-22T22:12:02.335481+0000","last_clean":"2026-02-22T22:12:02.335481+0000","last_became_active":"2026-02-22T21:36:17.702758+0000","last_became_peered":"2026-02-22T21:36:17.702758+0000","last_unstale":"2026-02-22T22:12:02.335481+0000","last_undegraded":"2026-02-22T22:12:02.335481+0000","last_fullsized":"2026-02-22T22:12:02.335481+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:16.695655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:16.695655+0000","last_clean_scrub_stamp":"2026-02-22T21:36:16.695655+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T01:00:00.761061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033481199999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":910,"reported_epoch":454,"state":"active+clean","last_fresh":"2026-02-22T22:12:02.334876+0000","last_change":"2026-02-22T21:36:19.704343+0000","last_active":"2026-02-22T22:12:02.334876+0000","last_peered":"2026-02-22T22:12:02.334876+0000","last_clean":"2026-02-22T22:12:02.334876+0000","last_became_active":"2026-02-22T21:36:17.702784+0000","last_became_peered":"2026-02-22T21:36:17.702784+0000","last_unstale":"2026-02-22T22:12:02.334876+0000","last_undegraded":"2026-02-22T22:12:02.334876+0000","last_fullsized":"2026-02-22T22:12:02.334876+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:16.695655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:16.695655+0000","last_clean_scrub_stamp":"2026-02-22T21:36:16.695655+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T01:51:15.121856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000400474,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"16'2","reported_seq":912,"reported_epoch":454,"state":"active+clean","last_fresh":"2026-02-22T22:12:03.093192+0000","last_change":"2026-02-22T21:36:19.706373+0000","last_active":"2026-02-22T22:12:03.093192+0000","last_peered":"2026-02-22T22:12:03.093192+0000","last_clean":"2026-02-22T22:12:03.093192+0000","last_became_active":"2026-02-22T21:36:17.702836+0000","last_became_peered":"2026-02-22T21:36:17.702836+0000","last_unstale":"2026-02-22T22:12:03.093192+0000","last_undegraded":"2026-02-22T22:12:03.093192+0000","last_fullsized":"2026-02-22T22:12:03.093192+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:16.695655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:16.695655+0000","last_clean_scrub_stamp":"2026-02-22T21:36:16.695655+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T09:04:07.404872+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00089484299999999998,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":912,"reported_epoch":454,"state":"active+clean","last_fresh":"2026-02-22T22:12:01.506040+0000","last_change":"2026-02-22T21:36:19.749070+0000","last_active":"2026-02-22T22:12:01.506040+0000","last_peered":"2026-02-22T22:12:01.506040+0000","last_clean":"2026-02-22T22:12:01.506040+0000","last_became_active":"2026-02-22T21:36:17.702842+0000","last_became_peered":"2026-02-22T21:36:17.702842+0000","last_unstale":"2026-02-22T22:12:01.506040+0000","last_undegraded":"2026-02-22T22:12:01.506040+0000","last_fullsized":"2026-02-22T22:12:01.506040+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:16.695655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:16.695655+0000","last_clean_scrub_stamp":"2026-02-22T21:36:16.695655+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T08:47:34.298671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00023738200000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":913,"reported_epoch":454,"state":"active+clean","last_fresh":"2026-02-22T22:12:01.504892+0000","last_change":"2026-02-22T21:36:19.749053+0000","last_active":"2026-02-22T22:12:01.504892+0000","last_peered":"2026-02-22T22:12:01.504892+0000","last_clean":"2026-02-22T22:12:01.504892+0000","last_became_active":"2026-02-22T21:36:17.702618+0000","last_became_peered":"2026-02-22T21:36:17.702618+0000","last_unstale":"2026-02-22T22:12:01.504892+0000","last_undegraded":"2026-02-22T22:12:01.504892+0000","last_fullsized":"2026-02-22T22:12:01.504892+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:16.695655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:16.695655+0000","last_clean_scrub_stamp":"2026-02-22T21:36:16.695655+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T23:00:17.618828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026033399999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"14'1","reported_seq":911,"reported_epoch":454,"state":"active+clean","last_fresh":"2026-02-22T22:12:02.331103+0000","last_change":"2026-02-22T21:36:19.704451+0000","last_active":"2026-02-22T22:12:02.331103+0000","last_peered":"2026-02-22T22:12:02.331103+0000","last_clean":"2026-02-22T22:12:02.331103+0000","last_became_active":"2026-02-22T21:36:17.702257+0000","last_became_peered":"2026-02-22T21:36:17.702257+0000","last_unstale":"2026-02-22T22:12:02.331103+0000","last_undegraded":"2026-02-22T22:12:02.331103+0000","last_fullsized":"2026-02-22T22:12:02.331103+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:16.695655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:16.695655+0000","last_clean_scrub_stamp":"2026-02-22T21:36:16.695655+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T01:42:15.152043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00049669299999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"11'72","reported_seq":1002,"reported_epoch":454,"state":"active+clean","last_fresh":"2026-02-22T22:12:02.334763+0000","last_change":"2026-02-22T21:36:14.450229+0000","last_active":"2026-02-22T22:12:02.334763+0000","last_peered":"2026-02-22T22:12:02.334763+0000","last_clean":"2026-02-22T22:12:02.334763+0000","last_became_active":"2026-02-22T21:36:14.449500+0000","last_became_peered":"2026-02-22T21:36:14.449500+0000","last_unstale":"2026-02-22T22:12:02.334763+0000","last_undegraded":"2026-02-22T22:12:02.334763+0000","last_fullsized":"2026-02-22T22:12:02.334763+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:12.692166+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:12.692166+0000","last_clean_scrub_stamp":"2026-02-22T21:36:12.692166+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T02:36:42.881463+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":1180736,"data_compressed":19658,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738808,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":75032,"kb_used_data":3648,"kb_used_omap":1,"kb_used_meta":71358,"kb_avail":94296808,"statfs":{"total":96636764160,"available":96559931392,"internally_reserved":0,"allocated":3735552,"data_stored":1998059,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":73071046},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738809,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":70928,"kb_used_data":3708,"kb_used_omap":1,"kb_used_meta":67198,"kb_avail":94300912,"statfs":{"total":96636764160,"available":96564133888,"internally_reserved":0,"allocated":3796992,"data_stored":2588446,"data_compressed":9829,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1594,"internal_metadata":68811206},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738809,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":111124,"kb_used_data":3708,"kb_used_omap":1,"kb_used_meta":107390,"kb_avail":94260716,"statfs":{"total":96636764160,"available":96522973184,"internally_reserved":0,"allocated":3796992,"data_stored":2588446,"data_compressed":9829,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1594,"internal_metadata":109967814},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":9829,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":9829,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-22T22:12:34.158 INFO:tasks.ceph.ceph_manager.cluster1:clean! 2026-02-22T22:12:34.159 DEBUG:teuthology.orchestra.run.trial004:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2026-02-22T22:12:34.317 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T22:12:34.317 INFO:teuthology.orchestra.run.trial004.stderr:dumped all 2026-02-22T22:12:34.325 INFO:teuthology.orchestra.run.trial004.stdout:{"pg_ready":true,"pg_map":{"version":1543,"stamp":"2026-02-22T22:12:32.934308+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":257084,"kb_used_data":11064,"kb_used_omap":4,"kb_used_meta":245947,"kb_avail":282858436,"statfs":{"total":289910292480,"available":289647038464,"internally_reserved":0,"allocated":11329536,"data_stored":7174951,"data_compressed":19658,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":4782,"internal_metadata":251850066},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.000847"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":909,"reported_epoch":454,"state":"active+clean","last_fresh":"2026-02-22T22:12:02.334605+0000","last_change":"2026-02-22T21:36:19.704598+0000","last_active":"2026-02-22T22:12:02.334605+0000","last_peered":"2026-02-22T22:12:02.334605+0000","last_clean":"2026-02-22T22:12:02.334605+0000","last_became_active":"2026-02-22T21:36:17.702763+0000","last_became_peered":"2026-02-22T21:36:17.702763+0000","last_unstale":"2026-02-22T22:12:02.334605+0000","last_undegraded":"2026-02-22T22:12:02.334605+0000","last_fullsized":"2026-02-22T22:12:02.334605+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:16.695655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:16.695655+0000","last_clean_scrub_stamp":"2026-02-22T21:36:16.695655+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T21:59:04.606522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00062915900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":909,"reported_epoch":454,"state":"active+clean","last_fresh":"2026-02-22T22:12:02.336125+0000","last_change":"2026-02-22T21:36:19.704496+0000","last_active":"2026-02-22T22:12:02.336125+0000","last_peered":"2026-02-22T22:12:02.336125+0000","last_clean":"2026-02-22T22:12:02.336125+0000","last_became_active":"2026-02-22T21:36:17.703106+0000","last_became_peered":"2026-02-22T21:36:17.703106+0000","last_unstale":"2026-02-22T22:12:02.336125+0000","last_undegraded":"2026-02-22T22:12:02.336125+0000","last_fullsized":"2026-02-22T22:12:02.336125+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:16.695655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:16.695655+0000","last_clean_scrub_stamp":"2026-02-22T21:36:16.695655+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T03:56:26.669952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00053011499999999995,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":910,"reported_epoch":454,"state":"active+clean","last_fresh":"2026-02-22T22:12:02.335481+0000","last_change":"2026-02-22T21:36:19.704255+0000","last_active":"2026-02-22T22:12:02.335481+0000","last_peered":"2026-02-22T22:12:02.335481+0000","last_clean":"2026-02-22T22:12:02.335481+0000","last_became_active":"2026-02-22T21:36:17.702758+0000","last_became_peered":"2026-02-22T21:36:17.702758+0000","last_unstale":"2026-02-22T22:12:02.335481+0000","last_undegraded":"2026-02-22T22:12:02.335481+0000","last_fullsized":"2026-02-22T22:12:02.335481+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:16.695655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:16.695655+0000","last_clean_scrub_stamp":"2026-02-22T21:36:16.695655+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T01:00:00.761061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033481199999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":910,"reported_epoch":454,"state":"active+clean","last_fresh":"2026-02-22T22:12:02.334876+0000","last_change":"2026-02-22T21:36:19.704343+0000","last_active":"2026-02-22T22:12:02.334876+0000","last_peered":"2026-02-22T22:12:02.334876+0000","last_clean":"2026-02-22T22:12:02.334876+0000","last_became_active":"2026-02-22T21:36:17.702784+0000","last_became_peered":"2026-02-22T21:36:17.702784+0000","last_unstale":"2026-02-22T22:12:02.334876+0000","last_undegraded":"2026-02-22T22:12:02.334876+0000","last_fullsized":"2026-02-22T22:12:02.334876+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:16.695655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:16.695655+0000","last_clean_scrub_stamp":"2026-02-22T21:36:16.695655+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T01:51:15.121856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000400474,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"16'2","reported_seq":912,"reported_epoch":454,"state":"active+clean","last_fresh":"2026-02-22T22:12:03.093192+0000","last_change":"2026-02-22T21:36:19.706373+0000","last_active":"2026-02-22T22:12:03.093192+0000","last_peered":"2026-02-22T22:12:03.093192+0000","last_clean":"2026-02-22T22:12:03.093192+0000","last_became_active":"2026-02-22T21:36:17.702836+0000","last_became_peered":"2026-02-22T21:36:17.702836+0000","last_unstale":"2026-02-22T22:12:03.093192+0000","last_undegraded":"2026-02-22T22:12:03.093192+0000","last_fullsized":"2026-02-22T22:12:03.093192+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:16.695655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:16.695655+0000","last_clean_scrub_stamp":"2026-02-22T21:36:16.695655+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T09:04:07.404872+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00089484299999999998,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":912,"reported_epoch":454,"state":"active+clean","last_fresh":"2026-02-22T22:12:01.506040+0000","last_change":"2026-02-22T21:36:19.749070+0000","last_active":"2026-02-22T22:12:01.506040+0000","last_peered":"2026-02-22T22:12:01.506040+0000","last_clean":"2026-02-22T22:12:01.506040+0000","last_became_active":"2026-02-22T21:36:17.702842+0000","last_became_peered":"2026-02-22T21:36:17.702842+0000","last_unstale":"2026-02-22T22:12:01.506040+0000","last_undegraded":"2026-02-22T22:12:01.506040+0000","last_fullsized":"2026-02-22T22:12:01.506040+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:16.695655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:16.695655+0000","last_clean_scrub_stamp":"2026-02-22T21:36:16.695655+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T08:47:34.298671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00023738200000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":913,"reported_epoch":454,"state":"active+clean","last_fresh":"2026-02-22T22:12:01.504892+0000","last_change":"2026-02-22T21:36:19.749053+0000","last_active":"2026-02-22T22:12:01.504892+0000","last_peered":"2026-02-22T22:12:01.504892+0000","last_clean":"2026-02-22T22:12:01.504892+0000","last_became_active":"2026-02-22T21:36:17.702618+0000","last_became_peered":"2026-02-22T21:36:17.702618+0000","last_unstale":"2026-02-22T22:12:01.504892+0000","last_undegraded":"2026-02-22T22:12:01.504892+0000","last_fullsized":"2026-02-22T22:12:01.504892+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:16.695655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:16.695655+0000","last_clean_scrub_stamp":"2026-02-22T21:36:16.695655+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T23:00:17.618828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026033399999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"14'1","reported_seq":911,"reported_epoch":454,"state":"active+clean","last_fresh":"2026-02-22T22:12:02.331103+0000","last_change":"2026-02-22T21:36:19.704451+0000","last_active":"2026-02-22T22:12:02.331103+0000","last_peered":"2026-02-22T22:12:02.331103+0000","last_clean":"2026-02-22T22:12:02.331103+0000","last_became_active":"2026-02-22T21:36:17.702257+0000","last_became_peered":"2026-02-22T21:36:17.702257+0000","last_unstale":"2026-02-22T22:12:02.331103+0000","last_undegraded":"2026-02-22T22:12:02.331103+0000","last_fullsized":"2026-02-22T22:12:02.331103+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:16.695655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:16.695655+0000","last_clean_scrub_stamp":"2026-02-22T21:36:16.695655+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T01:42:15.152043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00049669299999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"11'72","reported_seq":1002,"reported_epoch":454,"state":"active+clean","last_fresh":"2026-02-22T22:12:02.334763+0000","last_change":"2026-02-22T21:36:14.450229+0000","last_active":"2026-02-22T22:12:02.334763+0000","last_peered":"2026-02-22T22:12:02.334763+0000","last_clean":"2026-02-22T22:12:02.334763+0000","last_became_active":"2026-02-22T21:36:14.449500+0000","last_became_peered":"2026-02-22T21:36:14.449500+0000","last_unstale":"2026-02-22T22:12:02.334763+0000","last_undegraded":"2026-02-22T22:12:02.334763+0000","last_fullsized":"2026-02-22T22:12:02.334763+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:12.692166+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:12.692166+0000","last_clean_scrub_stamp":"2026-02-22T21:36:12.692166+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T02:36:42.881463+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":1180736,"data_compressed":19658,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738808,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":75032,"kb_used_data":3648,"kb_used_omap":1,"kb_used_meta":71358,"kb_avail":94296808,"statfs":{"total":96636764160,"available":96559931392,"internally_reserved":0,"allocated":3735552,"data_stored":1998059,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":73071046},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738809,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":70928,"kb_used_data":3708,"kb_used_omap":1,"kb_used_meta":67198,"kb_avail":94300912,"statfs":{"total":96636764160,"available":96564133888,"internally_reserved":0,"allocated":3796992,"data_stored":2588446,"data_compressed":9829,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1594,"internal_metadata":68811206},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738809,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":111124,"kb_used_data":3708,"kb_used_omap":1,"kb_used_meta":107390,"kb_avail":94260716,"statfs":{"total":96636764160,"available":96522973184,"internally_reserved":0,"allocated":3796992,"data_stored":2588446,"data_compressed":9829,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1594,"internal_metadata":109967814},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":9829,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":9829,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-22T22:12:34.326 DEBUG:teuthology.orchestra.run.trial004:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd dump --format=json 2026-02-22T22:12:34.485 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T22:12:34.486 INFO:teuthology.orchestra.run.trial004.stdout:{"epoch":454,"fsid":"54a0d3ac-2ee3-469c-a716-4416b2f462d9","created":"2026-02-22T21:36:02.327898+0000","modified":"2026-02-22T22:12:01.502262+0000","last_up_change":"2026-02-22T21:36:10.637268+0000","last_in_change":"2026-02-22T21:36:02.895351+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-02-22T21:36:12.642081+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2026-02-22T21:36:16.499800+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":16,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ad69c917-7b9d-4a6d-bd05-3187fef2ebef","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":27,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.4:6811","nonce":157985355},{"type":"v1","addr":"10.20.193.4:6813","nonce":157985355}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.4:6815","nonce":157985355},{"type":"v1","addr":"10.20.193.4:6818","nonce":157985355}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.4:6824","nonce":157985355},{"type":"v1","addr":"10.20.193.4:6825","nonce":157985355}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.4:6821","nonce":157985355},{"type":"v1","addr":"10.20.193.4:6823","nonce":157985355}]},"public_addr":"10.20.193.4:6813/157985355","cluster_addr":"10.20.193.4:6818/157985355","heartbeat_back_addr":"10.20.193.4:6825/157985355","heartbeat_front_addr":"10.20.193.4:6823/157985355","state":["exists","up"]},{"osd":1,"uuid":"8e27aac6-4677-41a2-8575-2b92ead46c7e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":27,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.4:6810","nonce":1030571860},{"type":"v1","addr":"10.20.193.4:6812","nonce":1030571860}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.4:6814","nonce":1030571860},{"type":"v1","addr":"10.20.193.4:6816","nonce":1030571860}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.4:6820","nonce":1030571860},{"type":"v1","addr":"10.20.193.4:6822","nonce":1030571860}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.4:6817","nonce":1030571860},{"type":"v1","addr":"10.20.193.4:6819","nonce":1030571860}]},"public_addr":"10.20.193.4:6812/1030571860","cluster_addr":"10.20.193.4:6816/1030571860","heartbeat_back_addr":"10.20.193.4:6822/1030571860","heartbeat_front_addr":"10.20.193.4:6819/1030571860","state":["exists","up"]},{"osd":2,"uuid":"28f6fce7-f1df-448c-91e9-b0eb54f0ff5f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":27,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.4:6800","nonce":283052529},{"type":"v1","addr":"10.20.193.4:6801","nonce":283052529}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.4:6802","nonce":283052529},{"type":"v1","addr":"10.20.193.4:6803","nonce":283052529}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.4:6806","nonce":283052529},{"type":"v1","addr":"10.20.193.4:6807","nonce":283052529}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.4:6804","nonce":283052529},{"type":"v1","addr":"10.20.193.4:6805","nonce":283052529}]},"public_addr":"10.20.193.4:6801/283052529","cluster_addr":"10.20.193.4:6803/283052529","heartbeat_back_addr":"10.20.193.4:6807/283052529","heartbeat_front_addr":"10.20.193.4:6805/283052529","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T21:36:09.625984+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T21:36:09.570742+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T21:36:09.368412+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-22T22:12:35.495 INFO:tasks.ceph:Scrubbing cluster1.osd.0 2026-02-22T22:12:35.496 DEBUG:teuthology.orchestra.run.trial004:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.0 config set osd_debug_deep_scrub_sleep 0 2026-02-22T22:12:35.576 INFO:teuthology.orchestra.run.trial004.stdout:{ 2026-02-22T22:12:35.576 INFO:teuthology.orchestra.run.trial004.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) osd_delete_sleep = '' osd_delete_sleep_hdd = '' osd_delete_sleep_hybrid = '' osd_delete_sleep_ssd = '' osd_max_backfills = '' osd_recovery_max_active = '' osd_recovery_max_active_hdd = '' osd_recovery_max_active_ssd = '' osd_recovery_sleep = '' osd_recovery_sleep_degraded = '' osd_recovery_sleep_degraded_hdd = '' osd_recovery_sleep_degraded_hybrid = '' osd_recovery_sleep_degraded_ssd = '' osd_recovery_sleep_hdd = '' osd_recovery_sleep_hybrid = '' osd_recovery_sleep_ssd = '' osd_scrub_sleep = '' osd_snap_trim_sleep = '' osd_snap_trim_sleep_hdd = '' osd_snap_trim_sleep_hybrid = '' osd_snap_trim_sleep_ssd = '' " 2026-02-22T22:12:35.576 INFO:teuthology.orchestra.run.trial004.stdout:} 2026-02-22T22:12:35.581 DEBUG:teuthology.orchestra.run.trial004:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd deep-scrub 0 2026-02-22T22:12:35.739 INFO:teuthology.orchestra.run.trial004.stderr:instructed osd(s) 0 to deep-scrub 2026-02-22T22:12:35.749 INFO:tasks.ceph:Scrubbing cluster1.osd.1 2026-02-22T22:12:35.749 DEBUG:teuthology.orchestra.run.trial004:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.1 config set osd_debug_deep_scrub_sleep 0 2026-02-22T22:12:35.829 INFO:teuthology.orchestra.run.trial004.stdout:{ 2026-02-22T22:12:35.829 INFO:teuthology.orchestra.run.trial004.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) osd_delete_sleep = '' osd_delete_sleep_hdd = '' osd_delete_sleep_hybrid = '' osd_delete_sleep_ssd = '' osd_max_backfills = '' osd_recovery_max_active = '' osd_recovery_max_active_hdd = '' osd_recovery_max_active_ssd = '' osd_recovery_sleep = '' osd_recovery_sleep_degraded = '' osd_recovery_sleep_degraded_hdd = '' osd_recovery_sleep_degraded_hybrid = '' osd_recovery_sleep_degraded_ssd = '' osd_recovery_sleep_hdd = '' osd_recovery_sleep_hybrid = '' osd_recovery_sleep_ssd = '' osd_scrub_sleep = '' osd_snap_trim_sleep = '' osd_snap_trim_sleep_hdd = '' osd_snap_trim_sleep_hybrid = '' osd_snap_trim_sleep_ssd = '' " 2026-02-22T22:12:35.829 INFO:teuthology.orchestra.run.trial004.stdout:} 2026-02-22T22:12:35.835 DEBUG:teuthology.orchestra.run.trial004:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd deep-scrub 1 2026-02-22T22:12:35.995 INFO:teuthology.orchestra.run.trial004.stderr:instructed osd(s) 1 to deep-scrub 2026-02-22T22:12:36.003 INFO:tasks.ceph:Scrubbing cluster1.osd.2 2026-02-22T22:12:36.004 DEBUG:teuthology.orchestra.run.trial004:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.2 config set osd_debug_deep_scrub_sleep 0 2026-02-22T22:12:36.084 INFO:teuthology.orchestra.run.trial004.stdout:{ 2026-02-22T22:12:36.085 INFO:teuthology.orchestra.run.trial004.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) osd_delete_sleep = '' osd_delete_sleep_hdd = '' osd_delete_sleep_hybrid = '' osd_delete_sleep_ssd = '' osd_max_backfills = '' osd_recovery_max_active = '' osd_recovery_max_active_hdd = '' osd_recovery_max_active_ssd = '' osd_recovery_sleep = '' osd_recovery_sleep_degraded = '' osd_recovery_sleep_degraded_hdd = '' osd_recovery_sleep_degraded_hybrid = '' osd_recovery_sleep_degraded_ssd = '' osd_recovery_sleep_hdd = '' osd_recovery_sleep_hybrid = '' osd_recovery_sleep_ssd = '' osd_scrub_sleep = '' osd_snap_trim_sleep = '' osd_snap_trim_sleep_hdd = '' osd_snap_trim_sleep_hybrid = '' osd_snap_trim_sleep_ssd = '' " 2026-02-22T22:12:36.085 INFO:teuthology.orchestra.run.trial004.stdout:} 2026-02-22T22:12:36.090 DEBUG:teuthology.orchestra.run.trial004:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd deep-scrub 2 2026-02-22T22:12:36.248 INFO:teuthology.orchestra.run.trial004.stderr:instructed osd(s) 2 to deep-scrub 2026-02-22T22:12:36.258 DEBUG:teuthology.orchestra.run.trial004:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2026-02-22T22:12:36.417 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T22:12:36.418 INFO:teuthology.orchestra.run.trial004.stderr:dumped all 2026-02-22T22:12:36.425 INFO:teuthology.orchestra.run.trial004.stdout:{"pg_ready":true,"pg_map":{"version":1544,"stamp":"2026-02-22T22:12:34.934562+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":257084,"kb_used_data":11064,"kb_used_omap":4,"kb_used_meta":245947,"kb_avail":282858436,"statfs":{"total":289910292480,"available":289647038464,"internally_reserved":0,"allocated":11329536,"data_stored":7174951,"data_compressed":19658,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":4782,"internal_metadata":251850066},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.000982"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":909,"reported_epoch":454,"state":"active+clean","last_fresh":"2026-02-22T22:12:02.334605+0000","last_change":"2026-02-22T21:36:19.704598+0000","last_active":"2026-02-22T22:12:02.334605+0000","last_peered":"2026-02-22T22:12:02.334605+0000","last_clean":"2026-02-22T22:12:02.334605+0000","last_became_active":"2026-02-22T21:36:17.702763+0000","last_became_peered":"2026-02-22T21:36:17.702763+0000","last_unstale":"2026-02-22T22:12:02.334605+0000","last_undegraded":"2026-02-22T22:12:02.334605+0000","last_fullsized":"2026-02-22T22:12:02.334605+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:16.695655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:16.695655+0000","last_clean_scrub_stamp":"2026-02-22T21:36:16.695655+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T21:59:04.606522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00062915900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":909,"reported_epoch":454,"state":"active+clean","last_fresh":"2026-02-22T22:12:02.336125+0000","last_change":"2026-02-22T21:36:19.704496+0000","last_active":"2026-02-22T22:12:02.336125+0000","last_peered":"2026-02-22T22:12:02.336125+0000","last_clean":"2026-02-22T22:12:02.336125+0000","last_became_active":"2026-02-22T21:36:17.703106+0000","last_became_peered":"2026-02-22T21:36:17.703106+0000","last_unstale":"2026-02-22T22:12:02.336125+0000","last_undegraded":"2026-02-22T22:12:02.336125+0000","last_fullsized":"2026-02-22T22:12:02.336125+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:16.695655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:16.695655+0000","last_clean_scrub_stamp":"2026-02-22T21:36:16.695655+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T03:56:26.669952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00053011499999999995,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":910,"reported_epoch":454,"state":"active+clean","last_fresh":"2026-02-22T22:12:02.335481+0000","last_change":"2026-02-22T21:36:19.704255+0000","last_active":"2026-02-22T22:12:02.335481+0000","last_peered":"2026-02-22T22:12:02.335481+0000","last_clean":"2026-02-22T22:12:02.335481+0000","last_became_active":"2026-02-22T21:36:17.702758+0000","last_became_peered":"2026-02-22T21:36:17.702758+0000","last_unstale":"2026-02-22T22:12:02.335481+0000","last_undegraded":"2026-02-22T22:12:02.335481+0000","last_fullsized":"2026-02-22T22:12:02.335481+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:16.695655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:16.695655+0000","last_clean_scrub_stamp":"2026-02-22T21:36:16.695655+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T01:00:00.761061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033481199999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":910,"reported_epoch":454,"state":"active+clean","last_fresh":"2026-02-22T22:12:02.334876+0000","last_change":"2026-02-22T21:36:19.704343+0000","last_active":"2026-02-22T22:12:02.334876+0000","last_peered":"2026-02-22T22:12:02.334876+0000","last_clean":"2026-02-22T22:12:02.334876+0000","last_became_active":"2026-02-22T21:36:17.702784+0000","last_became_peered":"2026-02-22T21:36:17.702784+0000","last_unstale":"2026-02-22T22:12:02.334876+0000","last_undegraded":"2026-02-22T22:12:02.334876+0000","last_fullsized":"2026-02-22T22:12:02.334876+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:16.695655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:16.695655+0000","last_clean_scrub_stamp":"2026-02-22T21:36:16.695655+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T01:51:15.121856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000400474,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"16'2","reported_seq":912,"reported_epoch":454,"state":"active+clean","last_fresh":"2026-02-22T22:12:03.093192+0000","last_change":"2026-02-22T21:36:19.706373+0000","last_active":"2026-02-22T22:12:03.093192+0000","last_peered":"2026-02-22T22:12:03.093192+0000","last_clean":"2026-02-22T22:12:03.093192+0000","last_became_active":"2026-02-22T21:36:17.702836+0000","last_became_peered":"2026-02-22T21:36:17.702836+0000","last_unstale":"2026-02-22T22:12:03.093192+0000","last_undegraded":"2026-02-22T22:12:03.093192+0000","last_fullsized":"2026-02-22T22:12:03.093192+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:16.695655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:16.695655+0000","last_clean_scrub_stamp":"2026-02-22T21:36:16.695655+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T09:04:07.404872+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00089484299999999998,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":912,"reported_epoch":454,"state":"active+clean","last_fresh":"2026-02-22T22:12:01.506040+0000","last_change":"2026-02-22T21:36:19.749070+0000","last_active":"2026-02-22T22:12:01.506040+0000","last_peered":"2026-02-22T22:12:01.506040+0000","last_clean":"2026-02-22T22:12:01.506040+0000","last_became_active":"2026-02-22T21:36:17.702842+0000","last_became_peered":"2026-02-22T21:36:17.702842+0000","last_unstale":"2026-02-22T22:12:01.506040+0000","last_undegraded":"2026-02-22T22:12:01.506040+0000","last_fullsized":"2026-02-22T22:12:01.506040+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:16.695655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:16.695655+0000","last_clean_scrub_stamp":"2026-02-22T21:36:16.695655+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T08:47:34.298671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00023738200000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":913,"reported_epoch":454,"state":"active+clean","last_fresh":"2026-02-22T22:12:01.504892+0000","last_change":"2026-02-22T21:36:19.749053+0000","last_active":"2026-02-22T22:12:01.504892+0000","last_peered":"2026-02-22T22:12:01.504892+0000","last_clean":"2026-02-22T22:12:01.504892+0000","last_became_active":"2026-02-22T21:36:17.702618+0000","last_became_peered":"2026-02-22T21:36:17.702618+0000","last_unstale":"2026-02-22T22:12:01.504892+0000","last_undegraded":"2026-02-22T22:12:01.504892+0000","last_fullsized":"2026-02-22T22:12:01.504892+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:16.695655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:16.695655+0000","last_clean_scrub_stamp":"2026-02-22T21:36:16.695655+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T23:00:17.618828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026033399999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"14'1","reported_seq":911,"reported_epoch":454,"state":"active+clean","last_fresh":"2026-02-22T22:12:02.331103+0000","last_change":"2026-02-22T21:36:19.704451+0000","last_active":"2026-02-22T22:12:02.331103+0000","last_peered":"2026-02-22T22:12:02.331103+0000","last_clean":"2026-02-22T22:12:02.331103+0000","last_became_active":"2026-02-22T21:36:17.702257+0000","last_became_peered":"2026-02-22T21:36:17.702257+0000","last_unstale":"2026-02-22T22:12:02.331103+0000","last_undegraded":"2026-02-22T22:12:02.331103+0000","last_fullsized":"2026-02-22T22:12:02.331103+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:16.695655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:16.695655+0000","last_clean_scrub_stamp":"2026-02-22T21:36:16.695655+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T01:42:15.152043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00049669299999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"11'72","reported_seq":1002,"reported_epoch":454,"state":"active+clean","last_fresh":"2026-02-22T22:12:02.334763+0000","last_change":"2026-02-22T21:36:14.450229+0000","last_active":"2026-02-22T22:12:02.334763+0000","last_peered":"2026-02-22T22:12:02.334763+0000","last_clean":"2026-02-22T22:12:02.334763+0000","last_became_active":"2026-02-22T21:36:14.449500+0000","last_became_peered":"2026-02-22T21:36:14.449500+0000","last_unstale":"2026-02-22T22:12:02.334763+0000","last_undegraded":"2026-02-22T22:12:02.334763+0000","last_fullsized":"2026-02-22T22:12:02.334763+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:36:12.692166+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:36:12.692166+0000","last_clean_scrub_stamp":"2026-02-22T21:36:12.692166+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T02:36:42.881463+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":1180736,"data_compressed":19658,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738808,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":75032,"kb_used_data":3648,"kb_used_omap":1,"kb_used_meta":71358,"kb_avail":94296808,"statfs":{"total":96636764160,"available":96559931392,"internally_reserved":0,"allocated":3735552,"data_stored":1998059,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":73071046},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738809,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":70928,"kb_used_data":3708,"kb_used_omap":1,"kb_used_meta":67198,"kb_avail":94300912,"statfs":{"total":96636764160,"available":96564133888,"internally_reserved":0,"allocated":3796992,"data_stored":2588446,"data_compressed":9829,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1594,"internal_metadata":68811206},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738809,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":111124,"kb_used_data":3708,"kb_used_omap":1,"kb_used_meta":107390,"kb_avail":94260716,"statfs":{"total":96636764160,"available":96522973184,"internally_reserved":0,"allocated":3796992,"data_stored":2588446,"data_compressed":9829,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1594,"internal_metadata":109967814},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":9829,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":9829,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-22T22:12:36.426 INFO:tasks.ceph:pgid 2.7 last_scrub_stamp 2026-02-22T21:36:16.695655+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=21, tm_min=36, tm_sec=16, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=12, tm_sec=34, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T22:12:36.426 INFO:tasks.ceph:pgid 2.6 last_scrub_stamp 2026-02-22T21:36:16.695655+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=21, tm_min=36, tm_sec=16, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=12, tm_sec=34, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T22:12:36.426 INFO:tasks.ceph:pgid 2.5 last_scrub_stamp 2026-02-22T21:36:16.695655+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=21, tm_min=36, tm_sec=16, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=12, tm_sec=34, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T22:12:36.426 INFO:tasks.ceph:pgid 2.4 last_scrub_stamp 2026-02-22T21:36:16.695655+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=21, tm_min=36, tm_sec=16, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=12, tm_sec=34, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T22:12:36.426 INFO:tasks.ceph:pgid 2.2 last_scrub_stamp 2026-02-22T21:36:16.695655+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=21, tm_min=36, tm_sec=16, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=12, tm_sec=34, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T22:12:36.426 INFO:tasks.ceph:pgid 2.1 last_scrub_stamp 2026-02-22T21:36:16.695655+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=21, tm_min=36, tm_sec=16, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=12, tm_sec=34, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T22:12:36.426 INFO:tasks.ceph:pgid 2.0 last_scrub_stamp 2026-02-22T21:36:16.695655+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=21, tm_min=36, tm_sec=16, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=12, tm_sec=34, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T22:12:36.426 INFO:tasks.ceph:pgid 2.3 last_scrub_stamp 2026-02-22T21:36:16.695655+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=21, tm_min=36, tm_sec=16, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=12, tm_sec=34, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T22:12:36.426 INFO:tasks.ceph:pgid 1.0 last_scrub_stamp 2026-02-22T21:36:12.692166+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=21, tm_min=36, tm_sec=12, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=12, tm_sec=34, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T22:12:36.426 INFO:tasks.ceph:Still waiting for all pgs to be scrubbed. 2026-02-22T22:12:37.721 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.2 has been restored 2026-02-22T22:12:37.721 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.mon.a has been restored 2026-02-22T22:12:56.427 DEBUG:teuthology.orchestra.run.trial004:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2026-02-22T22:12:56.586 INFO:teuthology.orchestra.run.trial004.stdout: 2026-02-22T22:12:56.586 INFO:teuthology.orchestra.run.trial004.stderr:dumped all 2026-02-22T22:12:56.595 INFO:teuthology.orchestra.run.trial004.stdout:{"pg_ready":true,"pg_map":{"version":1554,"stamp":"2026-02-22T22:12:54.936278+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":79,"ondisk_log_size":79,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":257064,"kb_used_data":11064,"kb_used_omap":4,"kb_used_meta":245947,"kb_avail":282858456,"statfs":{"total":289910292480,"available":289647058944,"internally_reserved":0,"allocated":11329536,"data_stored":7174951,"data_compressed":19658,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":4782,"internal_metadata":251850066},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.000921"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":918,"reported_epoch":454,"state":"active+clean","last_fresh":"2026-02-22T22:12:37.049722+0000","last_change":"2026-02-22T22:12:37.049735+0000","last_active":"2026-02-22T22:12:37.049722+0000","last_peered":"2026-02-22T22:12:37.049722+0000","last_clean":"2026-02-22T22:12:37.049722+0000","last_became_active":"2026-02-22T21:36:17.702763+0000","last_became_peered":"2026-02-22T21:36:17.702763+0000","last_unstale":"2026-02-22T22:12:37.049722+0000","last_undegraded":"2026-02-22T22:12:37.049722+0000","last_fullsized":"2026-02-22T22:12:37.049722+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:12:37.049675+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:12:37.049675+0000","last_clean_scrub_stamp":"2026-02-22T22:12:37.049675+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T02:01:13.394133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00062915900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":918,"reported_epoch":454,"state":"active+clean","last_fresh":"2026-02-22T22:12:41.015991+0000","last_change":"2026-02-22T22:12:41.016003+0000","last_active":"2026-02-22T22:12:41.015991+0000","last_peered":"2026-02-22T22:12:41.015991+0000","last_clean":"2026-02-22T22:12:41.015991+0000","last_became_active":"2026-02-22T21:36:17.703106+0000","last_became_peered":"2026-02-22T21:36:17.703106+0000","last_unstale":"2026-02-22T22:12:41.015991+0000","last_undegraded":"2026-02-22T22:12:41.015991+0000","last_fullsized":"2026-02-22T22:12:41.015991+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:12:41.015938+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:12:41.015938+0000","last_clean_scrub_stamp":"2026-02-22T22:12:41.015938+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T07:13:54.982159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00053011499999999995,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":919,"reported_epoch":454,"state":"active+clean","last_fresh":"2026-02-22T22:12:38.012309+0000","last_change":"2026-02-22T22:12:38.012318+0000","last_active":"2026-02-22T22:12:38.012309+0000","last_peered":"2026-02-22T22:12:38.012309+0000","last_clean":"2026-02-22T22:12:38.012309+0000","last_became_active":"2026-02-22T21:36:17.702758+0000","last_became_peered":"2026-02-22T21:36:17.702758+0000","last_unstale":"2026-02-22T22:12:38.012309+0000","last_undegraded":"2026-02-22T22:12:38.012309+0000","last_fullsized":"2026-02-22T22:12:38.012309+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:12:38.012276+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:12:38.012276+0000","last_clean_scrub_stamp":"2026-02-22T22:12:38.012276+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:46:41.911339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033481199999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":919,"reported_epoch":454,"state":"active+clean","last_fresh":"2026-02-22T22:12:40.043011+0000","last_change":"2026-02-22T22:12:40.043027+0000","last_active":"2026-02-22T22:12:40.043011+0000","last_peered":"2026-02-22T22:12:40.043011+0000","last_clean":"2026-02-22T22:12:40.043011+0000","last_became_active":"2026-02-22T21:36:17.702784+0000","last_became_peered":"2026-02-22T21:36:17.702784+0000","last_unstale":"2026-02-22T22:12:40.043011+0000","last_undegraded":"2026-02-22T22:12:40.043011+0000","last_fullsized":"2026-02-22T22:12:40.043011+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:12:40.042952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:12:40.042952+0000","last_clean_scrub_stamp":"2026-02-22T22:12:40.042952+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T08:36:11.123674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000400474,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"454'3","reported_seq":922,"reported_epoch":454,"state":"active+clean","last_fresh":"2026-02-22T22:12:36.331251+0000","last_change":"2026-02-22T22:12:36.331264+0000","last_active":"2026-02-22T22:12:36.331251+0000","last_peered":"2026-02-22T22:12:36.331251+0000","last_clean":"2026-02-22T22:12:36.331251+0000","last_became_active":"2026-02-22T21:36:17.702836+0000","last_became_peered":"2026-02-22T21:36:17.702836+0000","last_unstale":"2026-02-22T22:12:36.331251+0000","last_undegraded":"2026-02-22T22:12:36.331251+0000","last_fullsized":"2026-02-22T22:12:36.331251+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"454'3","last_scrub_stamp":"2026-02-22T22:12:36.331217+0000","last_deep_scrub":"454'3","last_deep_scrub_stamp":"2026-02-22T22:12:36.331217+0000","last_clean_scrub_stamp":"2026-02-22T22:12:36.331217+0000","objects_scrubbed":1,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T09:01:09.731774+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0.00089484299999999998,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":921,"reported_epoch":454,"state":"active+clean","last_fresh":"2026-02-22T22:12:37.547651+0000","last_change":"2026-02-22T22:12:37.547661+0000","last_active":"2026-02-22T22:12:37.547651+0000","last_peered":"2026-02-22T22:12:37.547651+0000","last_clean":"2026-02-22T22:12:37.547651+0000","last_became_active":"2026-02-22T21:36:17.702842+0000","last_became_peered":"2026-02-22T21:36:17.702842+0000","last_unstale":"2026-02-22T22:12:37.547651+0000","last_undegraded":"2026-02-22T22:12:37.547651+0000","last_fullsized":"2026-02-22T22:12:37.547651+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:12:37.547603+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:12:37.547603+0000","last_clean_scrub_stamp":"2026-02-22T22:12:37.547603+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T02:53:49.754164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00023738200000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":922,"reported_epoch":454,"state":"active+clean","last_fresh":"2026-02-22T22:12:36.573027+0000","last_change":"2026-02-22T22:12:36.573041+0000","last_active":"2026-02-22T22:12:36.573027+0000","last_peered":"2026-02-22T22:12:36.573027+0000","last_clean":"2026-02-22T22:12:36.573027+0000","last_became_active":"2026-02-22T21:36:17.702618+0000","last_became_peered":"2026-02-22T21:36:17.702618+0000","last_unstale":"2026-02-22T22:12:36.573027+0000","last_undegraded":"2026-02-22T22:12:36.573027+0000","last_fullsized":"2026-02-22T22:12:36.573027+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:12:36.572975+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:12:36.572975+0000","last_clean_scrub_stamp":"2026-02-22T22:12:36.572975+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T05:24:46.617777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026033399999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"454'2","reported_seq":921,"reported_epoch":454,"state":"active+clean","last_fresh":"2026-02-22T22:12:39.033410+0000","last_change":"2026-02-22T22:12:39.033426+0000","last_active":"2026-02-22T22:12:39.033410+0000","last_peered":"2026-02-22T22:12:39.033410+0000","last_clean":"2026-02-22T22:12:39.033410+0000","last_became_active":"2026-02-22T21:36:17.702257+0000","last_became_peered":"2026-02-22T21:36:17.702257+0000","last_unstale":"2026-02-22T22:12:39.033410+0000","last_undegraded":"2026-02-22T22:12:39.033410+0000","last_fullsized":"2026-02-22T22:12:39.033410+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"454'2","last_scrub_stamp":"2026-02-22T22:12:39.033363+0000","last_deep_scrub":"454'2","last_deep_scrub_stamp":"2026-02-22T22:12:39.033363+0000","last_clean_scrub_stamp":"2026-02-22T22:12:39.033363+0000","objects_scrubbed":1,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T23:11:13.213338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00049669299999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"454'74","reported_seq":1013,"reported_epoch":454,"state":"active+clean","last_fresh":"2026-02-22T22:12:36.039272+0000","last_change":"2026-02-22T22:12:36.039301+0000","last_active":"2026-02-22T22:12:36.039272+0000","last_peered":"2026-02-22T22:12:36.039272+0000","last_clean":"2026-02-22T22:12:36.039272+0000","last_became_active":"2026-02-22T21:36:14.449500+0000","last_became_peered":"2026-02-22T21:36:14.449500+0000","last_unstale":"2026-02-22T22:12:36.039272+0000","last_undegraded":"2026-02-22T22:12:36.039272+0000","last_fullsized":"2026-02-22T22:12:36.039272+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"454'74","last_scrub_stamp":"2026-02-22T22:12:36.039212+0000","last_deep_scrub":"454'74","last_deep_scrub_stamp":"2026-02-22T22:12:36.039212+0000","last_clean_scrub_stamp":"2026-02-22T22:12:36.039212+0000","objects_scrubbed":2,"log_size":74,"log_dups_size":0,"ondisk_log_size":74,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T07:50:19.002834+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":5,"ondisk_log_size":5,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":1180736,"data_compressed":19658,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":0,"internal_metadata":0},"log_size":74,"ondisk_log_size":74,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738812,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":75024,"kb_used_data":3648,"kb_used_omap":1,"kb_used_meta":71358,"kb_avail":94296816,"statfs":{"total":96636764160,"available":96559939584,"internally_reserved":0,"allocated":3735552,"data_stored":1998059,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":73071046},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738813,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":70924,"kb_used_data":3708,"kb_used_omap":1,"kb_used_meta":67198,"kb_avail":94300916,"statfs":{"total":96636764160,"available":96564137984,"internally_reserved":0,"allocated":3796992,"data_stored":2588446,"data_compressed":9829,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1594,"internal_metadata":68811206},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738813,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":111116,"kb_used_data":3708,"kb_used_omap":1,"kb_used_meta":107390,"kb_avail":94260724,"statfs":{"total":96636764160,"available":96522981376,"internally_reserved":0,"allocated":3796992,"data_stored":2588446,"data_compressed":9829,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1594,"internal_metadata":109967814},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":9829,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":9829,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-22T22:12:56.597 DEBUG:teuthology.orchestra.run.trial004:> sudo ceph --cluster cluster1 config set global mon_health_to_clog false 2026-02-22T22:12:56.766 INFO:teuthology.misc:Shutting down mds daemons... 2026-02-22T22:12:56.766 INFO:teuthology.misc:Shutting down osd daemons... 2026-02-22T22:12:56.766 DEBUG:tasks.ceph.cluster1.osd.0:waiting for process to exit 2026-02-22T22:12:56.766 INFO:teuthology.orchestra.run:waiting for 300 2026-02-22T22:12:56.806 INFO:tasks.ceph.cluster1.osd.0:Stopped 2026-02-22T22:12:56.806 DEBUG:tasks.ceph.cluster1.osd.1:waiting for process to exit 2026-02-22T22:12:56.806 INFO:teuthology.orchestra.run:waiting for 300 2026-02-22T22:12:56.884 INFO:tasks.ceph.cluster1.osd.1:Stopped 2026-02-22T22:12:56.885 DEBUG:tasks.ceph.cluster1.osd.2:waiting for process to exit 2026-02-22T22:12:56.885 INFO:teuthology.orchestra.run:waiting for 300 2026-02-22T22:12:56.955 INFO:tasks.ceph.cluster1.osd.2:Stopped 2026-02-22T22:12:56.956 INFO:teuthology.misc:Shutting down mgr daemons... 2026-02-22T22:12:56.956 DEBUG:tasks.ceph.cluster1.mgr.x:waiting for process to exit 2026-02-22T22:12:56.956 INFO:teuthology.orchestra.run:waiting for 300 2026-02-22T22:12:57.026 INFO:tasks.ceph.cluster1.mgr.x:Stopped 2026-02-22T22:12:57.026 INFO:teuthology.misc:Shutting down mon daemons... 2026-02-22T22:12:57.026 DEBUG:tasks.ceph.cluster1.mon.a:waiting for process to exit 2026-02-22T22:12:57.026 INFO:teuthology.orchestra.run:waiting for 300 2026-02-22T22:12:57.077 INFO:tasks.ceph.cluster1.mon.a:Stopped 2026-02-22T22:12:57.078 INFO:tasks.ceph:Checking cluster log for badness... 2026-02-22T22:12:57.078 DEBUG:teuthology.orchestra.run.trial004:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/cluster1.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v '\(OSD_SLOW_PING_TIME' | head -n 1 2026-02-22T22:12:57.146 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster1-0 on ubuntu@trial004.front.sepia.ceph.com 2026-02-22T22:12:57.146 DEBUG:teuthology.orchestra.run.trial004:> sync && sudo umount -f /var/lib/ceph/osd/cluster1-0 2026-02-22T22:12:57.217 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster1-1 on ubuntu@trial004.front.sepia.ceph.com 2026-02-22T22:12:57.217 DEBUG:teuthology.orchestra.run.trial004:> sync && sudo umount -f /var/lib/ceph/osd/cluster1-1 2026-02-22T22:12:57.289 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster1-2 on ubuntu@trial004.front.sepia.ceph.com 2026-02-22T22:12:57.289 DEBUG:teuthology.orchestra.run.trial004:> sync && sudo umount -f /var/lib/ceph/osd/cluster1-2 2026-02-22T22:12:57.362 INFO:tasks.ceph:Archiving mon data... 2026-02-22T22:12:57.362 DEBUG:teuthology.misc:Transferring archived files from trial004:/var/lib/ceph/mon/cluster1-a to /home/teuthworker/mnt/teuthology/yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial/64060/data/cluster1.mon.a.tgz 2026-02-22T22:12:57.362 DEBUG:teuthology.orchestra.run.trial004:> mktemp 2026-02-22T22:12:57.418 INFO:teuthology.orchestra.run.trial004.stdout:/tmp/tmp.07UVQcQtf1 2026-02-22T22:12:57.418 DEBUG:teuthology.orchestra.run.trial004:> sudo tar cz -f - -C /var/lib/ceph/mon/cluster1-a -- . > /tmp/tmp.07UVQcQtf1 2026-02-22T22:12:57.531 DEBUG:teuthology.orchestra.run.trial004:> sudo chmod 0666 /tmp/tmp.07UVQcQtf1 2026-02-22T22:12:57.613 DEBUG:teuthology.orchestra.remote:trial004:/tmp/tmp.07UVQcQtf1 is 383KB 2026-02-22T22:12:57.703 DEBUG:teuthology.orchestra.run.trial004:> rm -fr /tmp/tmp.07UVQcQtf1 2026-02-22T22:12:57.718 INFO:tasks.ceph:Cleaning ceph cluster... 2026-02-22T22:12:57.718 DEBUG:teuthology.orchestra.run.trial004:> sudo rm -rf -- /etc/ceph/cluster1.conf /etc/ceph/cluster1.keyring /home/ubuntu/cephtest/cluster1.data /home/ubuntu/cephtest/cluster1.monmap /home/ubuntu/cephtest/../*.pid 2026-02-22T22:12:57.760 DEBUG:teuthology.orchestra.run.trial026:> sudo rm -rf -- /etc/ceph/cluster1.conf /etc/ceph/cluster1.keyring /home/ubuntu/cephtest/cluster1.data /home/ubuntu/cephtest/cluster1.monmap /home/ubuntu/cephtest/../*.pid 2026-02-22T22:12:57.837 INFO:teuthology.util.scanner:summary_data or yaml_file is empty! 2026-02-22T22:12:57.852 INFO:teuthology.util.scanner:summary_data or yaml_file is empty! 2026-02-22T22:12:57.853 INFO:tasks.ceph:Archiving crash dumps... 2026-02-22T22:12:57.856 DEBUG:teuthology.misc:Transferring archived files from trial004:/var/lib/ceph/crash to /home/teuthworker/mnt/teuthology/yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial/64060/remote/trial004/crash 2026-02-22T22:12:57.857 DEBUG:teuthology.orchestra.run.trial004:> sudo tar c -f - -C /var/lib/ceph/crash -- . 2026-02-22T22:12:57.902 DEBUG:teuthology.misc:Transferring archived files from trial026:/var/lib/ceph/crash to /home/teuthworker/mnt/teuthology/yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial/64060/remote/trial026/crash 2026-02-22T22:12:57.904 DEBUG:teuthology.orchestra.run.trial026:> sudo tar c -f - -C /var/lib/ceph/crash -- . 2026-02-22T22:12:57.928 INFO:tasks.ceph:Compressing logs... 2026-02-22T22:12:57.928 DEBUG:teuthology.orchestra.run.trial004:> time sudo find /var/log/ceph -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2026-02-22T22:12:57.948 DEBUG:teuthology.orchestra.run.trial026:> time sudo find /var/log/ceph -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2026-02-22T22:12:57.970 INFO:teuthology.orchestra.run.trial004.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.45913.log 2026-02-22T22:12:57.971 INFO:teuthology.orchestra.run.trial004.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.48431.log 2026-02-22T22:12:57.971 INFO:teuthology.orchestra.run.trial004.stderr:/var/log/ceph/cluster1-client.admin.45913.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.48872.log 2026-02-22T22:12:57.971 INFO:teuthology.orchestra.run.trial004.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.45913.log.gz 2026-02-22T22:12:57.971 INFO:teuthology.orchestra.run.trial004.stderr:/var/log/ceph/cluster1-client.admin.48431.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.48431.log.gz 2026-02-22T22:12:57.971 INFO:teuthology.orchestra.run.trial004.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-mgr.x.log 2026-02-22T22:12:57.972 INFO:teuthology.orchestra.run.trial004.stderr:/var/log/ceph/cluster1-client.admin.48872.log: gzip -5 --verbose -- /var/log/ceph/cluster1-mon.a.log 2026-02-22T22:12:57.972 INFO:teuthology.orchestra.run.trial004.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.48872.log.gz 2026-02-22T22:12:57.972 INFO:teuthology.orchestra.run.trial004.stderr:/var/log/ceph/cluster2-mgr.x.log: gzip -5 --verbose -- /var/log/ceph/cluster1-osd.2.log 2026-02-22T22:12:57.972 INFO:teuthology.orchestra.run.trial004.stderr:/var/log/ceph/cluster1-mon.a.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.46179.log 2026-02-22T22:12:57.972 INFO:teuthology.orchestra.run.trial004.stderr:/var/log/ceph/cluster1-osd.2.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.48823.log 2026-02-22T22:12:57.973 INFO:teuthology.orchestra.run.trial004.stderr:/var/log/ceph/cluster1-client.admin.46179.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.46058.log 2026-02-22T22:12:57.973 INFO:teuthology.orchestra.run.trial004.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.46179.log.gz 2026-02-22T22:12:57.973 INFO:teuthology.orchestra.run.trial004.stderr:/var/log/ceph/cluster1-client.admin.48823.log: 0.0%gzip -- replaced with /var/log/ceph/cluster1-client.admin.48823.log.gz -5 2026-02-22T22:12:57.973 INFO:teuthology.orchestra.run.trial004.stderr: --verbose -- /var/log/ceph/cluster1-osd.0.log 2026-02-22T22:12:57.973 INFO:teuthology.orchestra.run.trial004.stderr:/var/log/ceph/cluster1-client.admin.46058.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.42689.log 2026-02-22T22:12:57.973 INFO:teuthology.orchestra.run.trial004.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.46058.log.gz 2026-02-22T22:12:57.973 INFO:teuthology.orchestra.run.trial004.stderr:/var/log/ceph/cluster1-osd.0.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.46180.log 2026-02-22T22:12:57.973 INFO:teuthology.orchestra.run.trial004.stderr:/var/log/ceph/cluster1-client.admin.42689.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.42689.log.gz 2026-02-22T22:12:57.974 INFO:teuthology.orchestra.run.trial004.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.45460.log 2026-02-22T22:12:57.974 INFO:teuthology.orchestra.run.trial004.stderr:/var/log/ceph/cluster1-client.admin.46180.log: gzip -5 --verbose -- /var/log/ceph/cluster1-osd.1.log 2026-02-22T22:12:57.974 INFO:teuthology.orchestra.run.trial004.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.46180.log.gz 2026-02-22T22:12:57.974 INFO:teuthology.orchestra.run.trial004.stderr:/var/log/ceph/cluster1-client.admin.45460.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.46398.log 2026-02-22T22:12:57.974 INFO:teuthology.orchestra.run.trial004.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.45460.log.gz 2026-02-22T22:12:57.974 INFO:teuthology.orchestra.run.trial004.stderr:/var/log/ceph/cluster1-osd.1.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.42880.log 2026-02-22T22:12:57.974 INFO:teuthology.orchestra.run.trial004.stderr:/var/log/ceph/cluster1-client.admin.46398.log: gzip 0.0% -5 -- replaced with /var/log/ceph/cluster1-client.admin.46398.log.gz --verbose 2026-02-22T22:12:57.974 INFO:teuthology.orchestra.run.trial004.stderr: -- /var/log/ceph/cluster1-client.admin.46300.log 2026-02-22T22:12:57.975 INFO:teuthology.orchestra.run.trial004.stderr:/var/log/ceph/cluster1-client.admin.42880.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.45761.log 2026-02-22T22:12:57.975 INFO:teuthology.orchestra.run.trial004.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.42880.log.gz 2026-02-22T22:12:57.975 INFO:teuthology.orchestra.run.trial004.stderr:/var/log/ceph/cluster1-client.admin.46300.log: gzip -5 --verbose -- /var/log/ceph/ceph.tmp-client.admin.37553.log 2026-02-22T22:12:57.975 INFO:teuthology.orchestra.run.trial004.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.46300.log.gz 2026-02-22T22:12:57.975 INFO:teuthology.orchestra.run.trial004.stderr:/var/log/ceph/cluster1-client.admin.45761.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.45761.log.gz 2026-02-22T22:12:57.975 INFO:teuthology.orchestra.run.trial004.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.48578.log 2026-02-22T22:12:57.975 INFO:teuthology.orchestra.run.trial004.stderr:/var/log/ceph/ceph.tmp-client.admin.37553.log: 0.0% -- replaced with /var/log/ceph/ceph.tmp-client.admin.37553.log.gz 2026-02-22T22:12:57.976 INFO:teuthology.orchestra.run.trial004.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.48529.log 2026-02-22T22:12:57.976 INFO:teuthology.orchestra.run.trial004.stderr:/var/log/ceph/cluster1-client.admin.48578.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.48578.log.gz 2026-02-22T22:12:57.976 INFO:teuthology.orchestra.run.trial004.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-mgr.x.log 2026-02-22T22:12:57.976 INFO:teuthology.orchestra.run.trial004.stderr:/var/log/ceph/cluster1-client.admin.48529.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.48529.log.gz 2026-02-22T22:12:57.976 INFO:teuthology.orchestra.run.trial004.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.46215.log 2026-02-22T22:12:57.976 INFO:teuthology.orchestra.run.trial004.stderr:/var/log/ceph/cluster1-mgr.x.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.48725.log 2026-02-22T22:12:57.976 INFO:teuthology.orchestra.run.trial004.stderr:/var/log/ceph/cluster1-client.admin.46215.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.46215.log.gz 2026-02-22T22:12:57.977 INFO:teuthology.orchestra.run.trial004.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.45712.log 2026-02-22T22:12:57.977 INFO:teuthology.orchestra.run.trial004.stderr:/var/log/ceph/cluster1-client.admin.48725.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.45647.log 2026-02-22T22:12:57.977 INFO:teuthology.orchestra.run.trial004.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.48725.log.gz 2026-02-22T22:12:57.977 INFO:teuthology.orchestra.run.trial004.stderr:/var/log/ceph/cluster1-client.admin.45712.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.45712.log.gz 2026-02-22T22:12:57.977 INFO:teuthology.orchestra.run.trial004.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.48382.log 2026-02-22T22:12:57.977 INFO:teuthology.orchestra.run.trial004.stderr:/var/log/ceph/cluster1-client.admin.45647.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.45622.log 2026-02-22T22:12:57.977 INFO:teuthology.orchestra.run.trial004.stderr: 80.4% -- replaced with /var/log/ceph/cluster1-client.admin.45647.log.gz 2026-02-22T22:12:57.977 INFO:teuthology.orchestra.run.trial004.stderr:/var/log/ceph/cluster1-client.admin.48382.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.48382.log.gz 2026-02-22T22:12:57.978 INFO:teuthology.orchestra.run.trial004.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.45911.log 2026-02-22T22:12:57.978 INFO:teuthology.orchestra.run.trial004.stderr:/var/log/ceph/cluster1-client.admin.45622.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.45622.log.gz 2026-02-22T22:12:57.978 INFO:teuthology.orchestra.run.trial004.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.42642.log 2026-02-22T22:12:57.978 INFO:teuthology.orchestra.run.trial004.stderr:/var/log/ceph/cluster1-client.admin.45911.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.48919.log 2026-02-22T22:12:57.978 INFO:teuthology.orchestra.run.trial004.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.45911.log.gz 2026-02-22T22:12:57.978 INFO:teuthology.orchestra.run.trial004.stderr:/var/log/ceph/cluster1-client.admin.42642.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.42642.log.gz 2026-02-22T22:12:57.978 INFO:teuthology.orchestra.run.trial004.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.46059.log 2026-02-22T22:12:57.978 INFO:teuthology.orchestra.run.trial004.stderr:/var/log/ceph/cluster1-client.admin.48919.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.48919.log.gz 2026-02-22T22:12:57.979 INFO:teuthology.orchestra.run.trial004.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.48333.log 2026-02-22T22:12:57.979 INFO:teuthology.orchestra.run.trial004.stderr:/var/log/ceph/cluster1-client.admin.46059.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.42595.log 2026-02-22T22:12:57.979 INFO:teuthology.orchestra.run.trial004.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.46059.log.gz 2026-02-22T22:12:57.979 INFO:teuthology.orchestra.run.trial004.stderr:/var/log/ceph/cluster1-client.admin.48333.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.48333.log.gz 2026-02-22T22:12:57.979 INFO:teuthology.orchestra.run.trial004.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.45810.log 2026-02-22T22:12:57.979 INFO:teuthology.orchestra.run.trial004.stderr:/var/log/ceph/cluster1-client.admin.42595.log: 0.0%gzip -- replaced with /var/log/ceph/cluster1-client.admin.42595.log.gz 2026-02-22T22:12:57.979 INFO:teuthology.orchestra.run.trial004.stderr: -5 --verbose -- /var/log/ceph/cluster1-client.admin.45912.log 2026-02-22T22:12:57.980 INFO:teuthology.orchestra.run.trial004.stderr:/var/log/ceph/cluster1-client.admin.45810.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.45810.log.gz 2026-02-22T22:12:57.980 INFO:teuthology.orchestra.run.trial004.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.48676.log 2026-02-22T22:12:57.980 INFO:teuthology.orchestra.run.trial004.stderr:/var/log/ceph/cluster1-client.admin.45912.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.45912.log.gz 2026-02-22T22:12:57.980 INFO:teuthology.orchestra.run.trial004.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1.audit.log 2026-02-22T22:12:57.980 INFO:teuthology.orchestra.run.trial004.stderr:/var/log/ceph/cluster1-client.admin.48676.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.48676.log.gz 2026-02-22T22:12:57.980 INFO:teuthology.orchestra.run.trial004.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.48627.log 2026-02-22T22:12:57.980 INFO:teuthology.orchestra.run.trial004.stderr:/var/log/ceph/cluster1.audit.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.46060.log 2026-02-22T22:12:57.981 INFO:teuthology.orchestra.run.trial004.stderr:/var/log/ceph/cluster1-client.admin.48627.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.48627.log.gz 2026-02-22T22:12:57.981 INFO:teuthology.orchestra.run.trial004.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.42410.log 2026-02-22T22:12:57.981 INFO:teuthology.orchestra.run.trial004.stderr:/var/log/ceph/cluster1-client.admin.46060.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.46060.log.gz 2026-02-22T22:12:57.981 INFO:teuthology.orchestra.run.trial004.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.46349.log 2026-02-22T22:12:57.981 INFO:teuthology.orchestra.run.trial004.stderr:/var/log/ceph/cluster1-client.admin.42410.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.42410.log.gz 2026-02-22T22:12:57.981 INFO:teuthology.orchestra.run.trial004.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.45552.log 2026-02-22T22:12:57.981 INFO:teuthology.orchestra.run.trial004.stderr:/var/log/ceph/cluster1-client.admin.46349.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.46349.log.gz 2026-02-22T22:12:57.982 INFO:teuthology.orchestra.run.trial004.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1.log 2026-02-22T22:12:57.982 INFO:teuthology.orchestra.run.trial004.stderr:/var/log/ceph/cluster1-client.admin.45552.log: 93.1% -- replaced with /var/log/ceph/cluster1.audit.log.gz 2026-02-22T22:12:57.982 INFO:teuthology.orchestra.run.trial004.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.45552.log.gz 2026-02-22T22:12:57.982 INFO:teuthology.orchestra.run.trial004.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.48480.log 2026-02-22T22:12:57.982 INFO:teuthology.orchestra.run.trial004.stderr:/var/log/ceph/cluster1.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.48774.log 2026-02-22T22:12:57.982 INFO:teuthology.orchestra.run.trial004.stderr:/var/log/ceph/cluster1-client.admin.48480.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.48480.log.gz 2026-02-22T22:12:57.982 INFO:teuthology.orchestra.run.trial004.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.42816.log 2026-02-22T22:12:57.982 INFO:teuthology.orchestra.run.trial004.stderr:/var/log/ceph/cluster1-client.admin.48774.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.48774.log.gz 2026-02-22T22:12:57.983 INFO:teuthology.orchestra.run.trial004.stderr:/var/log/ceph/cluster1-client.admin.42816.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.42816.log.gz 2026-02-22T22:12:57.984 INFO:teuthology.orchestra.run.trial004.stderr: 88.1% -- replaced with /var/log/ceph/cluster1.log.gz 2026-02-22T22:12:57.992 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.80118.log 2026-02-22T22:12:57.993 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78408.log 2026-02-22T22:12:57.993 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.80118.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68801.log 2026-02-22T22:12:57.993 INFO:teuthology.orchestra.run.trial026.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.80118.log.gz 2026-02-22T22:12:57.993 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.78408.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.78408.log.gz 2026-02-22T22:12:57.993 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.85031.log 2026-02-22T22:12:57.994 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70323.log 2026-02-22T22:12:57.994 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.68801.log: /var/log/ceph/cluster2-client.admin.85031.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.68801.log.gz 2026-02-22T22:12:57.994 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- 0.0% /var/log/ceph/cluster2-client.mirror.51289.log 2026-02-22T22:12:57.994 INFO:teuthology.orchestra.run.trial026.stderr: -- replaced with /var/log/ceph/cluster2-client.admin.85031.log.gz 2026-02-22T22:12:57.994 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.70323.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52723.log 2026-02-22T22:12:57.994 INFO:teuthology.orchestra.run.trial026.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.70323.log.gz 2026-02-22T22:12:57.994 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.51289.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.74133.log 2026-02-22T22:12:57.995 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.52723.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.80748.log 2026-02-22T22:12:57.995 INFO:teuthology.orchestra.run.trial026.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.51289.log.gz 2026-02-22T22:12:57.995 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.74133.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.82387.log 2026-02-22T22:12:57.995 INFO:teuthology.orchestra.run.trial026.stderr: 88.0% -- replaced with /var/log/ceph/cluster1-client.mirror.74133.log.gz 2026-02-22T22:12:57.995 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.80748.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56816.log 2026-02-22T22:12:57.995 INFO:teuthology.orchestra.run.trial026.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.80748.log.gz 2026-02-22T22:12:57.995 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.82387.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77813.log 2026-02-22T22:12:57.996 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.56816.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.82387.log.gz 2026-02-22T22:12:57.996 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57129.log 2026-02-22T22:12:57.996 INFO:teuthology.orchestra.run.trial026.stderr: 88.4% -- replaced with /var/log/ceph/cluster1-client.mirror.56816.log.gz 2026-02-22T22:12:57.996 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.77813.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.48636.log 2026-02-22T22:12:57.996 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.57129.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.77813.log.gz 2026-02-22T22:12:57.996 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70016.log 2026-02-22T22:12:57.997 INFO:teuthology.orchestra.run.trial026.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.57129.log.gz 2026-02-22T22:12:57.997 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.admin.48636.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83972.log 2026-02-22T22:12:57.997 INFO:teuthology.orchestra.run.trial026.stderr: 67.2% -- replaced with /var/log/ceph/cluster1-client.admin.48636.log.gz 2026-02-22T22:12:57.997 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.70016.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81711.log 2026-02-22T22:12:57.997 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.83972.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68603.log 2026-02-22T22:12:57.997 INFO:teuthology.orchestra.run.trial026.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.70016.log.gz 2026-02-22T22:12:57.997 INFO:teuthology.orchestra.run.trial026.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.83972.log.gz 2026-02-22T22:12:57.997 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.81711.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50595.log 2026-02-22T22:12:57.998 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.72282.log 2026-02-22T22:12:57.998 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.68603.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.81711.log.gz 2026-02-22T22:12:57.998 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.50595.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59949.log 2026-02-22T22:12:57.998 INFO:teuthology.orchestra.run.trial026.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.50595.log.gz 2026-02-22T22:12:57.998 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.72282.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75996.log 2026-02-22T22:12:57.998 INFO:teuthology.orchestra.run.trial026.stderr: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.68603.log.gz 2026-02-22T22:12:57.998 INFO:teuthology.orchestra.run.trial026.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.72282.log.gz 2026-02-22T22:12:57.998 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.59949.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79758.log 2026-02-22T22:12:57.999 INFO:teuthology.orchestra.run.trial026.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.59949.log.gz 2026-02-22T22:12:57.999 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.75996.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56839.log 2026-02-22T22:12:57.999 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.79758.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.75996.log.gz 2026-02-22T22:12:57.999 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.83544.log 2026-02-22T22:12:57.999 INFO:teuthology.orchestra.run.trial026.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.79758.log.gz 2026-02-22T22:12:57.999 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.56839.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75252.log 2026-02-22T22:12:57.999 INFO:teuthology.orchestra.run.trial026.stderr: 88.4% -- replaced with /var/log/ceph/cluster1-client.mirror.56839.log.gz 2026-02-22T22:12:58.000 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.83544.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57003.log 2026-02-22T22:12:58.000 INFO:teuthology.orchestra.run.trial026.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.83544.log.gz 2026-02-22T22:12:58.000 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.75252.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70528.log 2026-02-22T22:12:58.000 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.57003.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58076.log 2026-02-22T22:12:58.000 INFO:teuthology.orchestra.run.trial026.stderr: 88.1% 79.6% -- replaced with /var/log/ceph/cluster2-client.mirror.75252.log.gz -- replaced with /var/log/ceph/cluster2-client.mirror.57003.log.gz 2026-02-22T22:12:58.000 INFO:teuthology.orchestra.run.trial026.stderr: 2026-02-22T22:12:58.000 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.70528.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55308.log 2026-02-22T22:12:58.001 INFO:teuthology.orchestra.run.trial026.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.70528.log.gz 2026-02-22T22:12:58.001 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.58076.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58865.log 2026-02-22T22:12:58.001 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67337.log 2026-02-22T22:12:58.001 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.55308.log: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.58076.log.gz 2026-02-22T22:12:58.001 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.58865.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77957.log 2026-02-22T22:12:58.001 INFO:teuthology.orchestra.run.trial026.stderr: 89.6% -- replaced with /var/log/ceph/cluster1-client.mirror.55308.log.gz 2026-02-22T22:12:58.002 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.67337.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.82834.log 2026-02-22T22:12:58.002 INFO:teuthology.orchestra.run.trial026.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.67337.log.gz 89.6% 2026-02-22T22:12:58.002 INFO:teuthology.orchestra.run.trial026.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.58865.log.gz 2026-02-22T22:12:58.002 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.77957.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.74085.log 2026-02-22T22:12:58.002 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.82834.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70600.log 2026-02-22T22:12:58.002 INFO:teuthology.orchestra.run.trial026.stderr: 82.1% -- replaced with /var/log/ceph/cluster1-client.mirror.82834.log.gz 2026-02-22T22:12:58.003 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.74085.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52636.log 2026-02-22T22:12:58.003 INFO:teuthology.orchestra.run.trial026.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.74085.log.gz 2026-02-22T22:12:58.003 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.70600.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78700.log 2026-02-22T22:12:58.003 INFO:teuthology.orchestra.run.trial026.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.70600.log.gz 2026-02-22T22:12:58.003 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.52636.log: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.77957.log.gz 2026-02-22T22:12:58.003 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54233.log 2026-02-22T22:12:58.003 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.78700.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.46648.log 2026-02-22T22:12:58.003 INFO:teuthology.orchestra.run.trial026.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.78700.log.gz 2026-02-22T22:12:58.004 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.54233.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60692.log 2026-02-22T22:12:58.004 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.admin.46648.log: 80.4% -- replaced with /var/log/ceph/cluster2-client.admin.46648.log.gz 2026-02-22T22:12:58.004 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54212.log 2026-02-22T22:12:58.004 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.60692.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.71663.log 2026-02-22T22:12:58.004 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.54212.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74746.log 2026-02-22T22:12:58.004 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.71663.log: 88.5% -- replaced with /var/log/ceph/cluster1-client.mirror.54212.log.gz 2026-02-22T22:12:58.004 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.54857.log 2026-02-22T22:12:58.005 INFO:teuthology.orchestra.run.trial026.stderr: 88.0% -- replaced with /var/log/ceph/cluster1-client.mirror.71663.log.gz 2026-02-22T22:12:58.005 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.74746.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65030.log 2026-02-22T22:12:58.005 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.admin.54857.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56150.log 2026-02-22T22:12:58.005 INFO:teuthology.orchestra.run.trial026.stderr: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.52636.log.gz 2026-02-22T22:12:58.005 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.65030.log: 88.3% -- replaced with /var/log/ceph/cluster1-client.admin.54857.log.gzgzip 2026-02-22T22:12:58.005 INFO:teuthology.orchestra.run.trial026.stderr: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71684.log 2026-02-22T22:12:58.006 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.56150.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.46623.log 2026-02-22T22:12:58.006 INFO:teuthology.orchestra.run.trial026.stderr: 93.6% -- replaced with /var/log/ceph/cluster2-client.mirror.54233.log.gz 2026-02-22T22:12:58.006 INFO:teuthology.orchestra.run.trial026.stderr: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.74746.log.gz 2026-02-22T22:12:58.006 INFO:teuthology.orchestra.run.trial026.stderr: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.65030.log.gz 2026-02-22T22:12:58.006 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.71684.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52595.log 2026-02-22T22:12:58.006 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.admin.46623.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.46623.log.gz 2026-02-22T22:12:58.006 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73569.log 2026-02-22T22:12:58.007 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.52595.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76593.log 2026-02-22T22:12:58.007 INFO:teuthology.orchestra.run.trial026.stderr: 88.3% -- replaced with /var/log/ceph/cluster1-client.mirror.52595.log.gz 2026-02-22T22:12:58.007 INFO:teuthology.orchestra.run.trial026.stderr: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.71684.log.gz 2026-02-22T22:12:58.007 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.73569.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55260.log 2026-02-22T22:12:58.007 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.76593.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76779.log 2026-02-22T22:12:58.007 INFO:teuthology.orchestra.run.trial026.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.73569.log.gz 2026-02-22T22:12:58.007 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.55260.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68777.log 2026-02-22T22:12:58.008 INFO:teuthology.orchestra.run.trial026.stderr: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.55260.log.gz 2026-02-22T22:12:58.008 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.76779.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79645.log 2026-02-22T22:12:58.008 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.68777.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59246.log 2026-02-22T22:12:58.008 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.79645.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.68777.log.gz 2026-02-22T22:12:58.008 INFO:teuthology.orchestra.run.trial026.stderr: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.76593.log.gz 2026-02-22T22:12:58.008 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.52391.log 2026-02-22T22:12:58.008 INFO:teuthology.orchestra.run.trial026.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.79645.log.gz 2026-02-22T22:12:58.009 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.59246.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58280.log 2026-02-22T22:12:58.009 INFO:teuthology.orchestra.run.trial026.stderr: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.76779.log.gz 2026-02-22T22:12:58.009 INFO:teuthology.orchestra.run.trial026.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.59246.log.gz 2026-02-22T22:12:58.009 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.admin.52391.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58571.log 2026-02-22T22:12:58.009 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.58280.log: 88.0% -- replaced with /var/log/ceph/cluster1-client.admin.52391.log.gz 2026-02-22T22:12:58.009 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53551.log 2026-02-22T22:12:58.009 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.58571.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.58280.log.gz 2026-02-22T22:12:58.009 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77627.log 2026-02-22T22:12:58.010 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.53551.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78070.log 2026-02-22T22:12:58.010 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.77627.log: 89.0% -- replaced with /var/log/ceph/cluster1-client.mirror.53551.log.gz 2026-02-22T22:12:58.010 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70829.log 2026-02-22T22:12:58.010 INFO:teuthology.orchestra.run.trial026.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.77627.log.gz 2026-02-22T22:12:58.010 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.78070.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80349.log 2026-02-22T22:12:58.010 INFO:teuthology.orchestra.run.trial026.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.78070.log.gz 2026-02-22T22:12:58.011 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.70829.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83400.log 2026-02-22T22:12:58.011 INFO:teuthology.orchestra.run.trial026.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.70829.log.gz 2026-02-22T22:12:58.011 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.80349.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82195.log 2026-02-22T22:12:58.011 INFO:teuthology.orchestra.run.trial026.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.80349.log.gz 2026-02-22T22:12:58.011 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.83400.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58445.log 2026-02-22T22:12:58.011 INFO:teuthology.orchestra.run.trial026.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.83400.log.gz 2026-02-22T22:12:58.011 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.82195.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59099.log 2026-02-22T22:12:58.011 INFO:teuthology.orchestra.run.trial026.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.82195.log.gz 2026-02-22T22:12:58.011 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.58445.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70552.log 2026-02-22T22:12:58.012 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.59099.log: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.58445.log.gz 2026-02-22T22:12:58.012 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77441.log 2026-02-22T22:12:58.012 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.70552.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71991.log 2026-02-22T22:12:58.012 INFO:teuthology.orchestra.run.trial026.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.70552.log.gz 2026-02-22T22:12:58.012 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.77441.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75169.log 2026-02-22T22:12:58.012 INFO:teuthology.orchestra.run.trial026.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.77441.log.gz 2026-02-22T22:12:58.013 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.71991.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51835.log 2026-02-22T22:12:58.013 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.75169.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80371.log 2026-02-22T22:12:58.013 INFO:teuthology.orchestra.run.trial026.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.71991.log.gz 2026-02-22T22:12:58.013 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.51835.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.82640.log 2026-02-22T22:12:58.013 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.80371.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56042.log 2026-02-22T22:12:58.014 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.82640.log: gzip -5 --verbose -- /var/log/ceph/cluster2.log 2026-02-22T22:12:58.014 INFO:teuthology.orchestra.run.trial026.stderr: 90.3% -- replaced with /var/log/ceph/cluster2-client.mirror.75169.log.gz 2026-02-22T22:12:58.014 INFO:teuthology.orchestra.run.trial026.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.82640.log.gz 2026-02-22T22:12:58.014 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.56042.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.51998.log 2026-02-22T22:12:58.014 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66106.log 2026-02-22T22:12:58.014 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.admin.51998.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65056.log 2026-02-22T22:12:58.015 INFO:teuthology.orchestra.run.trial026.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.admin.51998.log.gz 2026-02-22T22:12:58.015 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.66106.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.64336.log 2026-02-22T22:12:58.015 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.65056.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.66106.log.gz 2026-02-22T22:12:58.015 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76037.log 2026-02-22T22:12:58.015 INFO:teuthology.orchestra.run.trial026.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.65056.log.gz 2026-02-22T22:12:58.015 INFO:teuthology.orchestra.run.trial026.stderr: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.51835.log.gz 2026-02-22T22:12:58.015 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.64336.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79488.log 2026-02-22T22:12:58.016 INFO:teuthology.orchestra.run.trial026.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.64336.log.gz 2026-02-22T22:12:58.016 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.76037.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64922.log 2026-02-22T22:12:58.016 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.79488.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51901.log 2026-02-22T22:12:58.016 INFO:teuthology.orchestra.run.trial026.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.79488.log.gz 2026-02-22T22:12:58.016 INFO:teuthology.orchestra.run.trial026.stderr: 93.6% -- replaced with /var/log/ceph/cluster2-client.mirror.56042.log.gz 2026-02-22T22:12:58.016 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.64922.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68074.log 2026-02-22T22:12:58.016 INFO:teuthology.orchestra.run.trial026.stderr: 87.3% -- replaced with /var/log/ceph/cluster2.log.gz 2026-02-22T22:12:58.016 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.51901.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69734.log 2026-02-22T22:12:58.016 INFO:teuthology.orchestra.run.trial026.stderr: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.76037.log.gz 2026-02-22T22:12:58.017 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.68074.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54116.log 2026-02-22T22:12:58.017 INFO:teuthology.orchestra.run.trial026.stderr: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.51901.log.gz 2026-02-22T22:12:58.017 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.69734.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.48166.log 2026-02-22T22:12:58.017 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.54116.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78048.log 2026-02-22T22:12:58.017 INFO:teuthology.orchestra.run.trial026.stderr: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.68074.log.gz 2026-02-22T22:12:58.017 INFO:teuthology.orchestra.run.trial026.stderr: 89.3% -- replaced with /var/log/ceph/cluster1-client.mirror.54116.log.gz 2026-02-22T22:12:58.018 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.48166.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60178.log 2026-02-22T22:12:58.018 INFO:teuthology.orchestra.run.trial026.stderr: 84.3% -- replaced with /var/log/ceph/cluster1-client.mirror.48166.log.gz 2026-02-22T22:12:58.018 INFO:teuthology.orchestra.run.trial026.stderr: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.69734.log.gz 2026-02-22T22:12:58.018 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.78048.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79983.log 2026-02-22T22:12:58.018 INFO:teuthology.orchestra.run.trial026.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.78048.log.gz 2026-02-22T22:12:58.018 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.60178.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60468.log 2026-02-22T22:12:58.018 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.79983.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57562.log 2026-02-22T22:12:58.018 INFO:teuthology.orchestra.run.trial026.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.79983.log.gz 2026-02-22T22:12:58.018 INFO:teuthology.orchestra.run.trial026.stderr: 95.3% -- replaced with /var/log/ceph/cluster1-client.mirror.60178.log.gz 2026-02-22T22:12:58.019 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.60468.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56527.log 2026-02-22T22:12:58.019 INFO:teuthology.orchestra.run.trial026.stderr: 79.6% -- replaced with /var/log/ceph/cluster2-client.mirror.60468.log.gz 2026-02-22T22:12:58.019 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.57562.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82595.log 2026-02-22T22:12:58.019 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.56527.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63957.log 2026-02-22T22:12:58.019 INFO:teuthology.orchestra.run.trial026.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.57562.log.gz 2026-02-22T22:12:58.019 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.82595.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.56527.log.gz 2026-02-22T22:12:58.019 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69267.log 2026-02-22T22:12:58.020 INFO:teuthology.orchestra.run.trial026.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.82595.log.gz 2026-02-22T22:12:58.020 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.63957.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.0.68940.log 2026-02-22T22:12:58.020 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.69267.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54972.log 2026-02-22T22:12:58.020 INFO:teuthology.orchestra.run.trial026.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.63957.log.gz 2026-02-22T22:12:58.020 INFO:teuthology.orchestra.run.trial026.stderr: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.69267.log.gz 2026-02-22T22:12:58.020 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.0.68940.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50571.log 2026-02-22T22:12:58.020 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.54972.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57301.log 2026-02-22T22:12:58.020 INFO:teuthology.orchestra.run.trial026.stderr: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.54972.log.gz 2026-02-22T22:12:58.021 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.50571.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56387.log 2026-02-22T22:12:58.021 INFO:teuthology.orchestra.run.trial026.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.50571.log.gz 2026-02-22T22:12:58.021 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.57301.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69525.log 2026-02-22T22:12:58.021 INFO:teuthology.orchestra.run.trial026.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.57301.log.gz 2026-02-22T22:12:58.021 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.56387.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58364.log 2026-02-22T22:12:58.021 INFO:teuthology.orchestra.run.trial026.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.56387.log.gz 2026-02-22T22:12:58.021 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.69525.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53806.log 2026-02-22T22:12:58.022 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.58364.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.48554.log 2026-02-22T22:12:58.022 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.53806.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53195.log 2026-02-22T22:12:58.022 INFO:teuthology.orchestra.run.trial026.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.58364.log.gz 2026-02-22T22:12:58.022 INFO:teuthology.orchestra.run.trial026.stderr: 92.5% -- replaced with /var/log/ceph/cluster1-client.mirror.69525.log.gz 2026-02-22T22:12:58.022 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.admin.48554.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.48554.log.gz 2026-02-22T22:12:58.022 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.82082.log 2026-02-22T22:12:58.022 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.53195.log: 91.7% -- replaced with /var/log/ceph/cluster2-client.mirror.53806.log.gz 2026-02-22T22:12:58.022 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.82452.log 2026-02-22T22:12:58.023 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.82082.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83565.log 2026-02-22T22:12:58.023 INFO:teuthology.orchestra.run.trial026.stderr: 86.6% -- replaced with /var/log/ceph/cluster1-client.mirror.82082.log.gz 2026-02-22T22:12:58.023 INFO:teuthology.orchestra.run.trial026.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.53195.log.gz 2026-02-22T22:12:58.023 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.82452.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55417.log 2026-02-22T22:12:58.023 INFO:teuthology.orchestra.run.trial026.stderr: 86.8% 93.8% -- replaced with /var/log/ceph/cluster1-client.mirror.82452.log.gz 2026-02-22T22:12:58.023 INFO:teuthology.orchestra.run.trial026.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.56150.log.gz 2026-02-22T22:12:58.023 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.83565.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75335.log 2026-02-22T22:12:58.023 INFO:teuthology.orchestra.run.trial026.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.83565.log.gz 2026-02-22T22:12:58.024 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.55417.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78565.log 2026-02-22T22:12:58.024 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.75335.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64443.log 2026-02-22T22:12:58.024 INFO:teuthology.orchestra.run.trial026.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.75335.log.gz 2026-02-22T22:12:58.024 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.78565.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64610.log 2026-02-22T22:12:58.024 INFO:teuthology.orchestra.run.trial026.stderr: 84.8% -- replaced with /var/log/ceph/cluster1-client.mirror.78565.log.gz 2026-02-22T22:12:58.024 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.64443.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.47219.log 2026-02-22T22:12:58.025 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.64610.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.72778.log 2026-02-22T22:12:58.025 INFO:teuthology.orchestra.run.trial026.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.64443.log.gz 2026-02-22T22:12:58.025 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.admin.47219.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51311.log 2026-02-22T22:12:58.025 INFO:teuthology.orchestra.run.trial026.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.47219.log.gz 2026-02-22T22:12:58.025 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.72778.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55044.log 2026-02-22T22:12:58.025 INFO:teuthology.orchestra.run.trial026.stderr: 92.7% 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.64610.log.gz -- replaced with /var/log/ceph/cluster1-client.mirror.72778.log.gz 2026-02-22T22:12:58.025 INFO:teuthology.orchestra.run.trial026.stderr: 2026-02-22T22:12:58.025 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.51311.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51087.log 2026-02-22T22:12:58.026 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.55044.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70755.log 2026-02-22T22:12:58.026 INFO:teuthology.orchestra.run.trial026.stderr: 89.4% -- replaced with /var/log/ceph/cluster1-client.mirror.55044.log.gz 2026-02-22T22:12:58.026 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.51087.log: 93.3% -- replaced with /var/log/ceph/cluster1-client.mirror.55417.log.gzgzip 2026-02-22T22:12:58.026 INFO:teuthology.orchestra.run.trial026.stderr: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66064.log 2026-02-22T22:12:58.026 INFO:teuthology.orchestra.run.trial026.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.51087.log.gz 2026-02-22T22:12:58.026 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.70755.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58118.log 2026-02-22T22:12:58.026 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.66064.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54946.log 2026-02-22T22:12:58.026 INFO:teuthology.orchestra.run.trial026.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.70755.log.gz 2026-02-22T22:12:58.027 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.58118.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72640.log 2026-02-22T22:12:58.027 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.54946.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67669.log 2026-02-22T22:12:58.027 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.72640.log: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.66064.log.gz 2026-02-22T22:12:58.027 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61830.log 2026-02-22T22:12:58.027 INFO:teuthology.orchestra.run.trial026.stderr: 90.1% -- replaced with /var/log/ceph/cluster2-client.mirror.54946.log.gz 2026-02-22T22:12:58.027 INFO:teuthology.orchestra.run.trial026.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.72640.log.gz 2026-02-22T22:12:58.027 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.67669.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66952.log 2026-02-22T22:12:58.028 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.61830.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81122.log 2026-02-22T22:12:58.028 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.66952.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.67669.log.gz 2026-02-22T22:12:58.028 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.48738.log 2026-02-22T22:12:58.028 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.81122.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61460.log 2026-02-22T22:12:58.028 INFO:teuthology.orchestra.run.trial026.stderr: 79.8% -- replaced with /var/log/ceph/cluster2-client.mirror.81122.log.gz 2026-02-22T22:12:58.028 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.48738.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.47302.log 2026-02-22T22:12:58.029 INFO:teuthology.orchestra.run.trial026.stderr: 57.1% -- replaced with /var/log/ceph/cluster1-client.mirror.48738.log.gz 2026-02-22T22:12:58.029 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62649.log 2026-02-22T22:12:58.029 INFO:teuthology.orchestra.run.trial026.stderr: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.66952.log.gz 2026-02-22T22:12:58.029 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.61460.log: /var/log/ceph/cluster2-client.admin.47302.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.47302.log.gz 2026-02-22T22:12:58.029 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84203.log 2026-02-22T22:12:58.029 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.62649.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73044.log 2026-02-22T22:12:58.029 INFO:teuthology.orchestra.run.trial026.stderr: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.61460.log.gz 2026-02-22T22:12:58.030 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.84203.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53640.log 2026-02-22T22:12:58.030 INFO:teuthology.orchestra.run.trial026.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.84203.log.gz 2026-02-22T22:12:58.030 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.73044.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70502.log 2026-02-22T22:12:58.030 INFO:teuthology.orchestra.run.trial026.stderr: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.62649.log.gz 2026-02-22T22:12:58.030 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63352.log 2026-02-22T22:12:58.030 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.53640.log: gzip -5/var/log/ceph/cluster2-client.mirror.70502.log: --verbose -- /var/log/ceph/cluster1-client.mirror.78295.log 2026-02-22T22:12:58.031 INFO:teuthology.orchestra.run.trial026.stderr: 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.73044.log.gz 2026-02-22T22:12:58.031 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.63352.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62462.log 2026-02-22T22:12:58.031 INFO:teuthology.orchestra.run.trial026.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.63352.log.gz 2026-02-22T22:12:58.031 INFO:teuthology.orchestra.run.trial026.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.70502.log.gz 2026-02-22T22:12:58.031 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53041.log 2026-02-22T22:12:58.031 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.78295.log: /var/log/ceph/cluster2-client.mirror.62462.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59315.log 2026-02-22T22:12:58.031 INFO:teuthology.orchestra.run.trial026.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.78295.log.gz 2026-02-22T22:12:58.031 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.53041.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74596.log 2026-02-22T22:12:58.032 INFO:teuthology.orchestra.run.trial026.stderr: 88.9% -- replaced with /var/log/ceph/cluster1-client.mirror.53041.log.gz 2026-02-22T22:12:58.032 INFO:teuthology.orchestra.run.trial026.stderr: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.62462.log.gz 2026-02-22T22:12:58.032 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.59315.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58263.log 2026-02-22T22:12:58.032 INFO:teuthology.orchestra.run.trial026.stderr: 79.6% -- replaced with /var/log/ceph/cluster2-client.mirror.59315.log.gz 2026-02-22T22:12:58.032 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.74596.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77792.log 2026-02-22T22:12:58.032 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.58263.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48093.log 2026-02-22T22:12:58.032 INFO:teuthology.orchestra.run.trial026.stderr: 79.4% -- replaced with /var/log/ceph/cluster2-client.mirror.58263.log.gz 2026-02-22T22:12:58.032 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.77792.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64357.log 2026-02-22T22:12:58.033 INFO:teuthology.orchestra.run.trial026.stderr: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.74596.log.gz 2026-02-22T22:12:58.033 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.48093.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.77792.log.gz 2026-02-22T22:12:58.033 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64240.log 2026-02-22T22:12:58.033 INFO:teuthology.orchestra.run.trial026.stderr: 84.5% -- replaced with /var/log/ceph/cluster2-client.mirror.48093.log.gz 2026-02-22T22:12:58.033 INFO:teuthology.orchestra.run.trial026.stderr: 94.0% -- replaced with /var/log/ceph/cluster2-client.mirror.0.68940.log.gz 2026-02-22T22:12:58.033 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.64357.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.0.53352.log 2026-02-22T22:12:58.033 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.64240.log: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.53640.log.gz 2026-02-22T22:12:58.033 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70241.log 2026-02-22T22:12:58.033 INFO:teuthology.orchestra.run.trial026.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.64240.log.gz 2026-02-22T22:12:58.034 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.0.53352.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54652.log 2026-02-22T22:12:58.034 INFO:teuthology.orchestra.run.trial026.stderr: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.64357.log.gz 2026-02-22T22:12:58.034 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.70241.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.72065.log 2026-02-22T22:12:58.034 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.54652.log: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.70241.log.gz 2026-02-22T22:12:58.034 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67219.log 2026-02-22T22:12:58.034 INFO:teuthology.orchestra.run.trial026.stderr: 89.2% -- replaced with /var/log/ceph/cluster1-client.mirror.54652.log.gz 2026-02-22T22:12:58.034 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75810.log 2026-02-22T22:12:58.035 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.72065.log: /var/log/ceph/cluster2-client.mirror.67219.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76399.log 2026-02-22T22:12:58.035 INFO:teuthology.orchestra.run.trial026.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.72065.log.gz 2026-02-22T22:12:58.035 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.75810.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78633.log 2026-02-22T22:12:58.035 INFO:teuthology.orchestra.run.trial026.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.67219.log.gz 2026-02-22T22:12:58.035 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.76399.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.75810.log.gz 2026-02-22T22:12:58.035 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.47694.log 2026-02-22T22:12:58.036 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.80028.log 2026-02-22T22:12:58.036 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.78633.log: /var/log/ceph/cluster1-client.admin.47694.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.78633.log.gz 2026-02-22T22:12:58.036 INFO:teuthology.orchestra.run.trial026.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.47694.log.gz 2026-02-22T22:12:58.036 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.63400.log 2026-02-22T22:12:58.036 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.80028.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.43699.log 2026-02-22T22:12:58.036 INFO:teuthology.orchestra.run.trial026.stderr: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.76399.log.gz 2026-02-22T22:12:58.036 INFO:teuthology.orchestra.run.trial026.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.80028.log.gz 2026-02-22T22:12:58.036 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.63400.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79600.log 2026-02-22T22:12:58.037 INFO:teuthology.orchestra.run.trial026.stderr: 93.3% -- replaced with /var/log/ceph/cluster2-client.mirror.0.53352.log.gz 2026-02-22T22:12:58.037 INFO:teuthology.orchestra.run.trial026.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.63400.log.gz 2026-02-22T22:12:58.037 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.admin.43699.log: gzip -5 0.0% --verbose -- replaced with /var/log/ceph/cluster2-client.admin.43699.log.gz -- 2026-02-22T22:12:58.037 INFO:teuthology.orchestra.run.trial026.stderr: /var/log/ceph/cluster1-client.mirror.63710.log 2026-02-22T22:12:58.037 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.79600.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54806.log 2026-02-22T22:12:58.037 INFO:teuthology.orchestra.run.trial026.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.79600.log.gz 2026-02-22T22:12:58.037 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.63710.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50489.log 2026-02-22T22:12:58.037 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.54806.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55332.log 2026-02-22T22:12:58.038 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.50489.log: 90.0% -- replaced with /var/log/ceph/cluster2-client.mirror.54806.log.gz 2026-02-22T22:12:58.038 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58059.log 2026-02-22T22:12:58.038 INFO:teuthology.orchestra.run.trial026.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.64922.log.gz 2026-02-22T22:12:58.038 INFO:teuthology.orchestra.run.trial026.stderr: 92.5% -- replaced with /var/log/ceph/cluster1-client.mirror.63710.log.gz 2026-02-22T22:12:58.038 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.55332.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78835.log 2026-02-22T22:12:58.038 INFO:teuthology.orchestra.run.trial026.stderr: 91.6% -- replaced with /var/log/ceph/cluster2-client.mirror.50489.log.gz 2026-02-22T22:12:58.038 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.58059.log: 88.7% -- replaced with /var/log/ceph/cluster1-client.mirror.55332.log.gz 2026-02-22T22:12:58.038 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61706.log 2026-02-22T22:12:58.039 INFO:teuthology.orchestra.run.trial026.stderr: 79.7% -- replaced with /var/log/ceph/cluster2-client.mirror.58059.log.gz 2026-02-22T22:12:58.039 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.78835.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.47062.log 2026-02-22T22:12:58.039 INFO:teuthology.orchestra.run.trial026.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.78835.log.gz 2026-02-22T22:12:58.039 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.61706.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79308.log 2026-02-22T22:12:58.039 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.admin.47062.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.83104.log 2026-02-22T22:12:58.039 INFO:teuthology.orchestra.run.trial026.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.47062.log.gz 2026-02-22T22:12:58.039 INFO:teuthology.orchestra.run.trial026.stderr: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.61706.log.gz 2026-02-22T22:12:58.039 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.79308.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73261.log 2026-02-22T22:12:58.040 INFO:teuthology.orchestra.run.trial026.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.79308.log.gz 2026-02-22T22:12:58.040 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.83104.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51063.log 2026-02-22T22:12:58.040 INFO:teuthology.orchestra.run.trial026.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.83104.log.gz 2026-02-22T22:12:58.040 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.73261.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82749.log 2026-02-22T22:12:58.040 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.51063.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.80926.log 2026-02-22T22:12:58.040 INFO:teuthology.orchestra.run.trial026.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.51063.log.gz 2026-02-22T22:12:58.040 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.82749.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57655.log 2026-02-22T22:12:58.041 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.80926.log: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.73261.log.gz 2026-02-22T22:12:58.041 INFO:teuthology.orchestra.run.trial026.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.82749.log.gz 2026-02-22T22:12:58.041 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.74527.log 2026-02-22T22:12:58.041 INFO:teuthology.orchestra.run.trial026.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.80926.log.gz 2026-02-22T22:12:58.041 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.57655.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77234.log 2026-02-22T22:12:58.041 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.74527.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53620.log 2026-02-22T22:12:58.041 INFO:teuthology.orchestra.run.trial026.stderr: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.57655.log.gz 2026-02-22T22:12:58.041 INFO:teuthology.orchestra.run.trial026.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.74527.log.gz 2026-02-22T22:12:58.042 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.77234.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76965.log 2026-02-22T22:12:58.042 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.53620.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.77234.log.gz 2026-02-22T22:12:58.042 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61983.log 2026-02-22T22:12:58.042 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.76965.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79915.log 2026-02-22T22:12:58.042 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.61983.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78498.log 2026-02-22T22:12:58.043 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.79915.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.79915.log.gz 2026-02-22T22:12:58.043 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59868.log 2026-02-22T22:12:58.043 INFO:teuthology.orchestra.run.trial026.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.61983.log.gz 2026-02-22T22:12:58.043 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60633.log 2026-02-22T22:12:58.043 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.78498.log: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.76965.log.gz 2026-02-22T22:12:58.043 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.59868.log: 86.6% -- replaced with /var/log/ceph/cluster1-client.mirror.78498.log.gz 2026-02-22T22:12:58.043 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48944.log 2026-02-22T22:12:58.043 INFO:teuthology.orchestra.run.trial026.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.59868.log.gz 2026-02-22T22:12:58.043 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.60633.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74642.log 2026-02-22T22:12:58.044 INFO:teuthology.orchestra.run.trial026.stderr: 79.8% -- replaced with /var/log/ceph/cluster2-client.mirror.60633.log.gz 2026-02-22T22:12:58.044 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.48944.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.2.48270.log 2026-02-22T22:12:58.044 INFO:teuthology.orchestra.run.trial026.stderr: 2.3% -- replaced with /var/log/ceph/cluster2-client.mirror.48944.log.gz 2026-02-22T22:12:58.044 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.74642.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66777.log 2026-02-22T22:12:58.044 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.2.48270.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.74642.log.gz 2026-02-22T22:12:58.044 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56347.log 2026-02-22T22:12:58.044 INFO:teuthology.orchestra.run.trial026.stderr: 93.6% -- replaced with /var/log/ceph/cluster2-client.mirror.53620.log.gz 2026-02-22T22:12:58.044 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.66777.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64835.log 2026-02-22T22:12:58.044 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.56347.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65798.log 2026-02-22T22:12:58.044 INFO:teuthology.orchestra.run.trial026.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.66777.log.gz 2026-02-22T22:12:58.045 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.64835.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.56347.log.gz 2026-02-22T22:12:58.045 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53017.log 2026-02-22T22:12:58.045 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.65798.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61111.log 2026-02-22T22:12:58.045 INFO:teuthology.orchestra.run.trial026.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.65798.log.gz 2026-02-22T22:12:58.045 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.53017.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84456.log 2026-02-22T22:12:58.045 INFO:teuthology.orchestra.run.trial026.stderr: 88.6% -- replaced with /var/log/ceph/cluster1-client.mirror.53017.log.gz 2026-02-22T22:12:58.045 INFO:teuthology.orchestra.run.trial026.stderr: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.64835.log.gz 2026-02-22T22:12:58.046 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.85423.log 2026-02-22T22:12:58.046 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.61111.log: /var/log/ceph/cluster1-client.mirror.84456.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51183.log 2026-02-22T22:12:58.046 INFO:teuthology.orchestra.run.trial026.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.61111.log.gz 2026-02-22T22:12:58.046 INFO:teuthology.orchestra.run.trial026.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.84456.log.gz 2026-02-22T22:12:58.046 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.admin.85423.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.85423.log.gz 2026-02-22T22:12:58.046 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70199.log 2026-02-22T22:12:58.046 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.51183.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75417.log 2026-02-22T22:12:58.046 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.70199.log: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.51183.log.gz 2026-02-22T22:12:58.046 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74254.log 2026-02-22T22:12:58.047 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.75417.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70436.log 2026-02-22T22:12:58.047 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.74254.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60568.log 2026-02-22T22:12:58.047 INFO:teuthology.orchestra.run.trial026.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.74254.log.gz 2026-02-22T22:12:58.047 INFO:teuthology.orchestra.run.trial026.stderr: 92.7% -- replaced with /var/log/ceph/cluster2-client.mirror.70199.log.gz 2026-02-22T22:12:58.047 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.70436.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84180.log 2026-02-22T22:12:58.048 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.60568.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51247.log 2026-02-22T22:12:58.048 INFO:teuthology.orchestra.run.trial026.stderr: 90.3% -- replaced with /var/log/ceph/cluster2-client.mirror.75417.log.gz 2026-02-22T22:12:58.048 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.84180.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50667.log 2026-02-22T22:12:58.048 INFO:teuthology.orchestra.run.trial026.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.84180.log.gz 2026-02-22T22:12:58.048 INFO:teuthology.orchestra.run.trial026.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.60568.log.gz 2026-02-22T22:12:58.048 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.51247.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54427.log 2026-02-22T22:12:58.048 INFO:teuthology.orchestra.run.trial026.stderr: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.70436.log.gz 2026-02-22T22:12:58.048 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.50667.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.63489.log 2026-02-22T22:12:58.048 INFO:teuthology.orchestra.run.trial026.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.50667.log.gz 2026-02-22T22:12:58.049 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.54427.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77089.log 2026-02-22T22:12:58.049 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.64152.log 2026-02-22T22:12:58.049 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.63489.log: /var/log/ceph/cluster2-client.mirror.77089.log: 91.7% -- replaced with /var/log/ceph/cluster2-client.mirror.54427.log.gz 2026-02-22T22:12:58.049 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56793.log 2026-02-22T22:12:58.049 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.64152.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52967.log 2026-02-22T22:12:58.050 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.56793.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60897.log 2026-02-22T22:12:58.050 INFO:teuthology.orchestra.run.trial026.stderr: 88.4% -- replaced with /var/log/ceph/cluster1-client.mirror.56793.log.gz 2026-02-22T22:12:58.050 INFO:teuthology.orchestra.run.trial026.stderr: 92.1% -- replaced with /var/log/ceph/cluster1-client.mirror.63489.log.gz 2026-02-22T22:12:58.050 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.52967.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56748.log 2026-02-22T22:12:58.050 INFO:teuthology.orchestra.run.trial026.stderr: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.77089.log.gz 2026-02-22T22:12:58.050 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.60897.log: 92.5% -- replaced with /var/log/ceph/cluster1-client.mirror.64152.log.gz 2026-02-22T22:12:58.050 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60734.log 2026-02-22T22:12:58.050 INFO:teuthology.orchestra.run.trial026.stderr: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.51247.log.gz 2026-02-22T22:12:58.051 INFO:teuthology.orchestra.run.trial026.stderr: 90.0% -- replaced with /var/log/ceph/cluster2-client.mirror.52967.log.gz 2026-02-22T22:12:58.051 INFO:teuthology.orchestra.run.trial026.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.60897.log.gz 2026-02-22T22:12:58.051 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.56748.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61646.log 2026-02-22T22:12:58.051 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.60734.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59890.log 2026-02-22T22:12:58.051 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.61646.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55176.log 2026-02-22T22:12:58.051 INFO:teuthology.orchestra.run.trial026.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.60734.log.gz 2026-02-22T22:12:58.051 INFO:teuthology.orchestra.run.trial026.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.61646.log.gz 2026-02-22T22:12:58.051 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.59890.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84710.log 2026-02-22T22:12:58.052 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.55176.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.62440.log 2026-02-22T22:12:58.052 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.84710.log: 91.0%gzip -- replaced with /var/log/ceph/cluster2-client.mirror.56748.log.gz -5 --verbose 2026-02-22T22:12:58.052 INFO:teuthology.orchestra.run.trial026.stderr: -- /var/log/ceph/cluster1-client.mirror.64312.log 2026-02-22T22:12:58.052 INFO:teuthology.orchestra.run.trial026.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.mirror.84710.log.gz 2026-02-22T22:12:58.052 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.62440.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71171.log 2026-02-22T22:12:58.053 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.64312.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.80163.log 2026-02-22T22:12:58.053 INFO:teuthology.orchestra.run.trial026.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.64312.log.gz 2026-02-22T22:12:58.053 INFO:teuthology.orchestra.run.trial026.stderr: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.62440.log.gz/var/log/ceph/cluster2-client.mirror.71171.log: 2026-02-22T22:12:58.053 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.53780.log 2026-02-22T22:12:58.053 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82558.log 2026-02-22T22:12:58.053 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.80163.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.71171.log.gz 2026-02-22T22:12:58.053 INFO:teuthology.orchestra.run.trial026.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.80163.log.gz 2026-02-22T22:12:58.054 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.admin.53780.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50688.log 2026-02-22T22:12:58.054 INFO:teuthology.orchestra.run.trial026.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.admin.53780.log.gz 2026-02-22T22:12:58.054 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.82558.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82038.log 2026-02-22T22:12:58.054 INFO:teuthology.orchestra.run.trial026.stderr: 79.9% -- replaced with /var/log/ceph/cluster2-client.mirror.82558.log.gz 2026-02-22T22:12:58.054 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.50688.log: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.55176.log.gz 2026-02-22T22:12:58.054 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79240.log 2026-02-22T22:12:58.054 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.82038.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79803.log 2026-02-22T22:12:58.055 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose --/var/log/ceph/cluster1-client.mirror.79240.log: /var/log/ceph/cluster2-client.mirror.56667.log 2026-02-22T22:12:58.055 INFO:teuthology.orchestra.run.trial026.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.79240.log.gz 2026-02-22T22:12:58.055 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.79803.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56062.log 2026-02-22T22:12:58.055 INFO:teuthology.orchestra.run.trial026.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.79803.log.gz 2026-02-22T22:12:58.055 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.56667.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81139.log 2026-02-22T22:12:58.056 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.56062.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.56667.log.gz 2026-02-22T22:12:58.056 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79038.log 2026-02-22T22:12:58.056 INFO:teuthology.orchestra.run.trial026.stderr: 91.6% -- replaced with /var/log/ceph/cluster2-client.mirror.82038.log.gz 2026-02-22T22:12:58.056 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78430.log 2026-02-22T22:12:58.056 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.81139.log: /var/log/ceph/cluster1-client.mirror.79038.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67034.log 2026-02-22T22:12:58.056 INFO:teuthology.orchestra.run.trial026.stderr: 86.6% -- replaced with /var/log/ceph/cluster1-client.mirror.79038.log.gz 2026-02-22T22:12:58.056 INFO:teuthology.orchestra.run.trial026.stderr: 93.4% -- replaced with /var/log/ceph/cluster2-client.mirror.50688.log.gz 2026-02-22T22:12:58.057 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.78430.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72176.log 2026-02-22T22:12:58.057 INFO:teuthology.orchestra.run.trial026.stderr: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.81139.log.gz 2026-02-22T22:12:58.057 INFO:teuthology.orchestra.run.trial026.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.78430.log.gz 2026-02-22T22:12:58.057 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.67034.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76183.log 2026-02-22T22:12:58.057 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.72176.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68029.log 2026-02-22T22:12:58.057 INFO:teuthology.orchestra.run.trial026.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.67034.log.gz 2026-02-22T22:12:58.057 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.76183.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.72176.log.gz 2026-02-22T22:12:58.057 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69754.log 2026-02-22T22:12:58.057 INFO:teuthology.orchestra.run.trial026.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.76183.log.gz 2026-02-22T22:12:58.058 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.71084.log 2026-02-22T22:12:58.058 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.68029.log: /var/log/ceph/cluster1-client.mirror.69754.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54741.log 2026-02-22T22:12:58.058 INFO:teuthology.orchestra.run.trial026.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.68029.log.gz 2026-02-22T22:12:58.058 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.71084.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49029.log 2026-02-22T22:12:58.058 INFO:teuthology.orchestra.run.trial026.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.71084.log.gz 2026-02-22T22:12:58.058 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.54741.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51923.log 2026-02-22T22:12:58.059 INFO:teuthology.orchestra.run.trial026.stderr: 92.5% -- replaced with /var/log/ceph/cluster1-client.mirror.69754.log.gz 2026-02-22T22:12:58.059 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.49029.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74371.log 2026-02-22T22:12:58.059 INFO:teuthology.orchestra.run.trial026.stderr: 93.6% -- replaced with /var/log/ceph/cluster1-client.mirror.56062.log.gz 2026-02-22T22:12:58.059 INFO:teuthology.orchestra.run.trial026.stderr: 54.9% -- replaced with /var/log/ceph/cluster2-client.mirror.49029.log.gz 2026-02-22T22:12:58.059 INFO:teuthology.orchestra.run.trial026.stderr:gzip/var/log/ceph/cluster2-client.mirror.51923.log: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55913.log 2026-02-22T22:12:58.059 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.74371.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.48772.log 2026-02-22T22:12:58.060 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.55913.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59182.log 2026-02-22T22:12:58.060 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.48772.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76635.log 2026-02-22T22:12:58.060 INFO:teuthology.orchestra.run.trial026.stderr: 54.5% -- replaced with /var/log/ceph/cluster1-client.mirror.48772.log.gz 2026-02-22T22:12:58.060 INFO:teuthology.orchestra.run.trial026.stderr: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.74371.log.gz 2026-02-22T22:12:58.060 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.59182.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61808.log 2026-02-22T22:12:58.061 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.76635.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56770.log 2026-02-22T22:12:58.061 INFO:teuthology.orchestra.run.trial026.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.76635.log.gz 2026-02-22T22:12:58.061 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.61808.log: gzip -5 93.5% --verbose -- -- replaced with /var/log/ceph/cluster1-client.mirror.54741.log.gz 2026-02-22T22:12:58.061 INFO:teuthology.orchestra.run.trial026.stderr: /var/log/ceph/cluster2-client.mirror.53479.log 2026-02-22T22:12:58.061 INFO:teuthology.orchestra.run.trial026.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.61808.log.gz 2026-02-22T22:12:58.061 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.56770.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59374.log 2026-02-22T22:12:58.061 INFO:teuthology.orchestra.run.trial026.stderr: 88.4% -- replaced with /var/log/ceph/cluster1-client.mirror.56770.log.gz 2026-02-22T22:12:58.062 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.53479.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.48858.log 2026-02-22T22:12:58.062 INFO:teuthology.orchestra.run.trial026.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.53479.log.gz 2026-02-22T22:12:58.062 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose/var/log/ceph/cluster2-client.mirror.59374.log: -- /var/log/ceph/cluster1-client.mirror.3.83609.log 2026-02-22T22:12:58.062 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.admin.48858.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60108.log 2026-02-22T22:12:58.062 INFO:teuthology.orchestra.run.trial026.stderr: 80.0% -- replaced with /var/log/ceph/cluster2-client.admin.48858.log.gz 2026-02-22T22:12:58.062 INFO:teuthology.orchestra.run.trial026.stderr: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.55913.log.gz 2026-02-22T22:12:58.063 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.3.83609.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66043.log 2026-02-22T22:12:58.063 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.60108.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84265.log 2026-02-22T22:12:58.063 INFO:teuthology.orchestra.run.trial026.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.60108.log.gz 2026-02-22T22:12:58.063 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.66043.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66883.log 2026-02-22T22:12:58.064 INFO:teuthology.orchestra.run.trial026.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.66043.log.gz 2026-02-22T22:12:58.064 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.84265.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72134.log 2026-02-22T22:12:58.064 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.66883.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62154.log 2026-02-22T22:12:58.064 INFO:teuthology.orchestra.run.trial026.stderr: 87.5% 93.1% -- replaced with /var/log/ceph/cluster1-client.mirror.66883.log.gz 2026-02-22T22:12:58.064 INFO:teuthology.orchestra.run.trial026.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.2.48270.log.gz 2026-02-22T22:12:58.064 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.72134.log: gzip -5 --verbose -- /var/log/ceph/cluster2.audit.log 2026-02-22T22:12:58.064 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.62154.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58885.log 2026-02-22T22:12:58.065 INFO:teuthology.orchestra.run.trial026.stderr: 2.6% -- replaced with /var/log/ceph/cluster2-client.mirror.62154.log.gz 2026-02-22T22:12:58.065 INFO:teuthology.orchestra.run.trial026.stderr: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.3.83609.log.gz 2026-02-22T22:12:58.065 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2.audit.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49780.log 2026-02-22T22:12:58.065 INFO:teuthology.orchestra.run.trial026.stderr: 91.6% -- replaced with /var/log/ceph/cluster2-client.mirror.84265.log.gz 2026-02-22T22:12:58.065 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.58885.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.63376.log 2026-02-22T22:12:58.065 INFO:teuthology.orchestra.run.trial026.stderr: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.72134.log.gz 2026-02-22T22:12:58.065 INFO:teuthology.orchestra.run.trial026.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.58885.log.gz 2026-02-22T22:12:58.065 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.49780.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79443.log 2026-02-22T22:12:58.066 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83204.log/var/log/ceph/cluster1-client.mirror.63376.log: 2026-02-22T22:12:58.066 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.79443.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.63376.log.gz 2026-02-22T22:12:58.066 INFO:teuthology.orchestra.run.trial026.stderr: 86.7%gzip -- replaced with /var/log/ceph/cluster1-client.mirror.79443.log.gz 2026-02-22T22:12:58.066 INFO:teuthology.orchestra.run.trial026.stderr: -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52106.log 2026-02-22T22:12:58.066 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.83204.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71346.log 2026-02-22T22:12:58.067 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.52106.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59400.log 2026-02-22T22:12:58.067 INFO:teuthology.orchestra.run.trial026.stderr: 88.3% -- replaced with /var/log/ceph/cluster1-client.mirror.52106.log.gz 2026-02-22T22:12:58.067 INFO:teuthology.orchestra.run.trial026.stderr: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.83204.log.gz 2026-02-22T22:12:58.067 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.71346.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77730.log 2026-02-22T22:12:58.067 INFO:teuthology.orchestra.run.trial026.stderr: 92.7% -- replaced with /var/log/ceph/cluster2.audit.log.gz 2026-02-22T22:12:58.067 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.59400.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.63175.log 2026-02-22T22:12:58.067 INFO:teuthology.orchestra.run.trial026.stderr: 79.6% -- replaced with /var/log/ceph/cluster2-client.mirror.59400.log.gz 2026-02-22T22:12:58.068 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.77730.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56467.log 2026-02-22T22:12:58.068 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.63175.log: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.71346.log.gz 2026-02-22T22:12:58.068 INFO:teuthology.orchestra.run.trial026.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.77730.log.gz 2026-02-22T22:12:58.068 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61966.log 2026-02-22T22:12:58.068 INFO:teuthology.orchestra.run.trial026.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.63175.log.gz 2026-02-22T22:12:58.068 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.56467.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49068.log 2026-02-22T22:12:58.069 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.61966.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51005.log 2026-02-22T22:12:58.069 INFO:teuthology.orchestra.run.trial026.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.56467.log.gz 2026-02-22T22:12:58.069 INFO:teuthology.orchestra.run.trial026.stderr: 79.7% -- replaced with /var/log/ceph/cluster2-client.mirror.61966.log.gz 2026-02-22T22:12:58.069 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.49068.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.3.82834.log 2026-02-22T22:12:58.069 INFO:teuthology.orchestra.run.trial026.stderr: 79.8% -- replaced with /var/log/ceph/cluster2-client.mirror.49068.log.gz 2026-02-22T22:12:58.069 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.51005.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55526.log 2026-02-22T22:12:58.069 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.62327.log 2026-02-22T22:12:58.070 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.3.82834.log: /var/log/ceph/cluster2-client.mirror.55526.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66972.log 2026-02-22T22:12:58.070 INFO:teuthology.orchestra.run.trial026.stderr: 91.7% -- replaced with /var/log/ceph/cluster2-client.mirror.51005.log.gz 2026-02-22T22:12:58.070 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.62327.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55284.log 2026-02-22T22:12:58.070 INFO:teuthology.orchestra.run.trial026.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.62327.log.gz/var/log/ceph/cluster1-client.mirror.66972.log: 2026-02-22T22:12:58.070 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62735.log 2026-02-22T22:12:58.071 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.55284.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.84562.log 2026-02-22T22:12:58.071 INFO:teuthology.orchestra.run.trial026.stderr: 89.5% -- replaced with /var/log/ceph/cluster1-client.mirror.55284.log.gz 2026-02-22T22:12:58.071 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.62735.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52571.log 2026-02-22T22:12:58.071 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.2.84562.log: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.66972.log.gz 2026-02-22T22:12:58.071 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78678.log 2026-02-22T22:12:58.071 INFO:teuthology.orchestra.run.trial026.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.62735.log.gz 2026-02-22T22:12:58.072 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.52571.log: 93.2% -- replaced with /var/log/ceph/cluster2-client.mirror.3.82834.log.gz 2026-02-22T22:12:58.072 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70178.log 2026-02-22T22:12:58.072 INFO:teuthology.orchestra.run.trial026.stderr: 88.5% -- replaced with /var/log/ceph/cluster1-client.mirror.52571.log.gz 2026-02-22T22:12:58.072 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.78678.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82617.log 2026-02-22T22:12:58.072 INFO:teuthology.orchestra.run.trial026.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.78678.log.gz 2026-02-22T22:12:58.072 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.70178.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52082.log 2026-02-22T22:12:58.072 INFO:teuthology.orchestra.run.trial026.stderr: 91.5% -- replaced with /var/log/ceph/cluster1-client.mirror.2.84562.log.gz 2026-02-22T22:12:58.072 INFO:teuthology.orchestra.run.trial026.stderr: 93.5%/var/log/ceph/cluster2-client.mirror.82617.log: -- replaced with /var/log/ceph/cluster2-client.mirror.55526.log.gz 2026-02-22T22:12:58.072 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70689.log 2026-02-22T22:12:58.073 INFO:teuthology.orchestra.run.trial026.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.70178.log.gz 2026-02-22T22:12:58.073 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.52082.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76697.log 2026-02-22T22:12:58.073 INFO:teuthology.orchestra.run.trial026.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.52082.log.gz 2026-02-22T22:12:58.073 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.70689.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60527.log 2026-02-22T22:12:58.073 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.76697.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60485.log 2026-02-22T22:12:58.074 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.60527.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84433.log 2026-02-22T22:12:58.074 INFO:teuthology.orchestra.run.trial026.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.76697.log.gz 2026-02-22T22:12:58.074 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.60485.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.62375.log 2026-02-22T22:12:58.074 INFO:teuthology.orchestra.run.trial026.stderr: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.70689.log.gz 2026-02-22T22:12:58.074 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.84433.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63511.log 2026-02-22T22:12:58.074 INFO:teuthology.orchestra.run.trial026.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.84433.log.gz 2026-02-22T22:12:58.074 INFO:teuthology.orchestra.run.trial026.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.60485.log.gz 2026-02-22T22:12:58.075 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.62375.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63915.log 2026-02-22T22:12:58.075 INFO:teuthology.orchestra.run.trial026.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.62375.log.gz 2026-02-22T22:12:58.075 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.63511.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55092.log 2026-02-22T22:12:58.075 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.63915.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83125.log 2026-02-22T22:12:58.075 INFO:teuthology.orchestra.run.trial026.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.63511.log.gz 2026-02-22T22:12:58.076 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.55092.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74154.log 2026-02-22T22:12:58.076 INFO:teuthology.orchestra.run.trial026.stderr: 88.6% -- replaced with /var/log/ceph/cluster1-client.mirror.55092.log.gz 2026-02-22T22:12:58.076 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.83125.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54066.log 2026-02-22T22:12:58.076 INFO:teuthology.orchestra.run.trial026.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.83125.log.gz 2026-02-22T22:12:58.076 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.74154.log: 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.63915.log.gz 2026-02-22T22:12:58.076 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55784.log 2026-02-22T22:12:58.077 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.54066.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58907.log 2026-02-22T22:12:58.077 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.55784.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84391.log 2026-02-22T22:12:58.077 INFO:teuthology.orchestra.run.trial026.stderr: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.54066.log.gz 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.74154.log.gz 2026-02-22T22:12:58.077 INFO:teuthology.orchestra.run.trial026.stderr: 2026-02-22T22:12:58.077 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.58907.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81604.log 2026-02-22T22:12:58.078 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.84391.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75893.log 2026-02-22T22:12:58.078 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.81604.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.84391.log.gz 2026-02-22T22:12:58.078 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60983.log 2026-02-22T22:12:58.078 INFO:teuthology.orchestra.run.trial026.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.81604.log.gz 2026-02-22T22:12:58.078 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.75893.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77213.log 2026-02-22T22:12:58.078 INFO:teuthology.orchestra.run.trial026.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.75893.log.gz 2026-02-22T22:12:58.078 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.60983.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58529.log 2026-02-22T22:12:58.079 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.77213.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61361.log 2026-02-22T22:12:58.079 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.58529.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64877.log 2026-02-22T22:12:58.080 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.61361.log: gzip -5 79.7% --verbose -- replaced with /var/log/ceph/cluster2-client.mirror.61361.log.gz -- 2026-02-22T22:12:58.080 INFO:teuthology.orchestra.run.trial026.stderr: /var/log/ceph/cluster2-client.mirror.60370.log 2026-02-22T22:12:58.080 INFO:teuthology.orchestra.run.trial026.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.58529.log.gz 2026-02-22T22:12:58.080 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.64877.log: 95.4% -- replaced with /var/log/ceph/cluster1-client.mirror.60983.log.gz 2026-02-22T22:12:58.080 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53683.log 2026-02-22T22:12:58.080 INFO:teuthology.orchestra.run.trial026.stderr: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.77213.log.gz 2026-02-22T22:12:58.080 INFO:teuthology.orchestra.run.trial026.stderr: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.55784.log.gz 2026-02-22T22:12:58.080 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.60370.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.64877.log.gz 2026-02-22T22:12:58.080 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74683.log 2026-02-22T22:12:58.081 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.53683.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79510.log 2026-02-22T22:12:58.081 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.74683.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78970.log 2026-02-22T22:12:58.081 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.79510.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76224.log 2026-02-22T22:12:58.081 INFO:teuthology.orchestra.run.trial026.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.79510.log.gz 2026-02-22T22:12:58.082 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.78970.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.73240.log 2026-02-22T22:12:58.082 INFO:teuthology.orchestra.run.trial026.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.78970.log.gz 2026-02-22T22:12:58.082 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.76224.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53173.log 2026-02-22T22:12:58.082 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.73240.log: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.74683.log.gz 2026-02-22T22:12:58.082 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.48876.log 2026-02-22T22:12:58.082 INFO:teuthology.orchestra.run.trial026.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.73240.log.gz 2026-02-22T22:12:58.083 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.53173.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76268.log 2026-02-22T22:12:58.083 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.admin.48876.log: gzip 80.5% -- replaced with /var/log/ceph/cluster2-client.admin.48876.log.gz 2026-02-22T22:12:58.083 INFO:teuthology.orchestra.run.trial026.stderr: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61626.log 2026-02-22T22:12:58.083 INFO:teuthology.orchestra.run.trial026.stderr: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.76224.log.gz 2026-02-22T22:12:58.083 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.76268.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.0.48166.log 2026-02-22T22:12:58.084 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.61626.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78363.log 2026-02-22T22:12:58.084 INFO:teuthology.orchestra.run.trial026.stderr: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.53683.log.gz 2026-02-22T22:12:58.084 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.0.48166.log: gzip 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.61626.log.gz 2026-02-22T22:12:58.084 INFO:teuthology.orchestra.run.trial026.stderr: -5 --verbose -- /var/log/ceph/cluster2-client.admin.85178.log 2026-02-22T22:12:58.085 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.78363.log: gzip -5 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.78363.log.gz 2026-02-22T22:12:58.085 INFO:teuthology.orchestra.run.trial026.stderr: --verbose -- /var/log/ceph/cluster2-client.mirror.59352.log 2026-02-22T22:12:58.085 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.admin.85178.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.85178.log.gz 2026-02-22T22:12:58.085 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose 91.9% -- replaced with /var/log/ceph/cluster2-client.mirror.76268.log.gz 2026-02-22T22:12:58.085 INFO:teuthology.orchestra.run.trial026.stderr: -- /var/log/ceph/cluster1-client.mirror.53599.log 2026-02-22T22:12:58.085 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.59352.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65947.log 2026-02-22T22:12:58.085 INFO:teuthology.orchestra.run.trial026.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.59352.log.gz 2026-02-22T22:12:58.086 INFO:teuthology.orchestra.run.trial026.stderr: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.53173.log.gz 2026-02-22T22:12:58.086 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.53599.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82302.log 2026-02-22T22:12:58.086 INFO:teuthology.orchestra.run.trial026.stderr: 91.7% -- replaced with /var/log/ceph/cluster2-client.mirror.0.48166.log.gz 88.3% -- replaced with /var/log/ceph/cluster1-client.mirror.53599.log.gz 2026-02-22T22:12:58.086 INFO:teuthology.orchestra.run.trial026.stderr: 2026-02-22T22:12:58.086 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.65947.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73303.log 2026-02-22T22:12:58.086 INFO:teuthology.orchestra.run.trial026.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.65947.log.gz 2026-02-22T22:12:58.086 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.82302.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60071.log 2026-02-22T22:12:58.087 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.73303.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.47060.log 2026-02-22T22:12:58.087 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.60071.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.82302.log.gz 2026-02-22T22:12:58.087 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 79.6% -- replaced with /var/log/ceph/cluster2-client.mirror.60071.log.gz 2026-02-22T22:12:58.087 INFO:teuthology.orchestra.run.trial026.stderr: --verbose -- /var/log/ceph/cluster2-client.mirror.62878.log 2026-02-22T22:12:58.087 INFO:teuthology.orchestra.run.trial026.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.73303.log.gz 2026-02-22T22:12:58.087 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.admin.47060.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.47060.log.gz 2026-02-22T22:12:58.088 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.53280.log 2026-02-22T22:12:58.088 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.62878.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.72730.log 2026-02-22T22:12:58.088 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.admin.53280.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.52796.log 2026-02-22T22:12:58.088 INFO:teuthology.orchestra.run.trial026.stderr: 88.1%/var/log/ceph/cluster1-client.mirror.72730.log: -- replaced with /var/log/ceph/cluster1-client.admin.53280.log.gz 2026-02-22T22:12:58.089 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55589.log 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.72730.log.gz 2026-02-22T22:12:58.089 INFO:teuthology.orchestra.run.trial026.stderr: 2026-02-22T22:12:58.089 INFO:teuthology.orchestra.run.trial026.stderr: 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.62878.log.gz 2026-02-22T22:12:58.089 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.admin.52796.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72901.log 2026-02-22T22:12:58.089 INFO:teuthology.orchestra.run.trial026.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.admin.52796.log.gz 2026-02-22T22:12:58.089 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.55589.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71203.log 2026-02-22T22:12:58.090 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.72901.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56327.log 2026-02-22T22:12:58.090 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.71203.log: gzip 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.72901.log.gz 2026-02-22T22:12:58.090 INFO:teuthology.orchestra.run.trial026.stderr: -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79060.log 2026-02-22T22:12:58.091 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.56327.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81690.log 2026-02-22T22:12:58.091 INFO:teuthology.orchestra.run.trial026.stderr: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.71203.log.gz 2026-02-22T22:12:58.091 INFO:teuthology.orchestra.run.trial026.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.56327.log.gz 2026-02-22T22:12:58.091 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.79060.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77275.log 2026-02-22T22:12:58.091 INFO:teuthology.orchestra.run.trial026.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.79060.log.gz 2026-02-22T22:12:58.091 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.81690.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.71639.log 2026-02-22T22:12:58.091 INFO:teuthology.orchestra.run.trial026.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.81690.log.gz 2026-02-22T22:12:58.092 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.77275.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61261.log 2026-02-22T22:12:58.092 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.71639.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.85129.log 2026-02-22T22:12:58.092 INFO:teuthology.orchestra.run.trial026.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.71639.log.gz 2026-02-22T22:12:58.092 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.61261.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59674.log 2026-02-22T22:12:58.093 INFO:teuthology.orchestra.run.trial004.stderr: 91.2% -- replaced with /var/log/ceph/cluster2-mgr.x.log.gz 2026-02-22T22:12:58.093 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.admin.85129.log: gzip -5 --verbose 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.55589.log.gz 2026-02-22T22:12:58.093 INFO:teuthology.orchestra.run.trial026.stderr: 28.5% -- replaced with /var/log/ceph/cluster2-client.admin.85129.log.gz 2026-02-22T22:12:58.093 INFO:teuthology.orchestra.run.trial026.stderr: -- /var/log/ceph/cluster1-client.mirror.63870.log 2026-02-22T22:12:58.093 INFO:teuthology.orchestra.run.trial026.stderr: 90.4%/var/log/ceph/cluster2-client.mirror.59674.log: -- replaced with /var/log/ceph/cluster2-client.mirror.77275.log.gz 2026-02-22T22:12:58.093 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.72331.log 2026-02-22T22:12:58.093 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.63870.log: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.59674.log.gz 2026-02-22T22:12:58.094 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78183.log 2026-02-22T22:12:58.094 INFO:teuthology.orchestra.run.trial026.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.63870.log.gz 2026-02-22T22:12:58.094 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.72331.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83993.log 2026-02-22T22:12:58.094 INFO:teuthology.orchestra.run.trial026.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.72331.log.gz/var/log/ceph/cluster1-client.mirror.78183.log: 2026-02-22T22:12:58.094 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.78183.log.gz 2026-02-22T22:12:58.094 INFO:teuthology.orchestra.run.trial026.stderr: --verbose -- /var/log/ceph/cluster2-client.mirror.62085.log 2026-02-22T22:12:58.095 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.83993.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72455.log 2026-02-22T22:12:58.095 INFO:teuthology.orchestra.run.trial026.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.83993.log.gz 2026-02-22T22:12:58.095 INFO:teuthology.orchestra.run.trial004.stderr: 91.3% -- replaced with /var/log/ceph/cluster1-mgr.x.log.gz 2026-02-22T22:12:58.095 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.62085.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.43920.log 2026-02-22T22:12:58.095 INFO:teuthology.orchestra.run.trial026.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.62085.log.gz 2026-02-22T22:12:58.095 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.72455.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.52769.log 2026-02-22T22:12:58.096 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.admin.43920.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.43920.log.gz 2026-02-22T22:12:58.096 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55847.log 2026-02-22T22:12:58.096 INFO:teuthology.orchestra.run.trial026.stderr: 89.8%/var/log/ceph/cluster1-client.admin.52769.log: -- replaced with /var/log/ceph/cluster2-client.mirror.72455.log.gz 2026-02-22T22:12:58.096 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.73888.log 2026-02-22T22:12:58.096 INFO:teuthology.orchestra.run.trial026.stderr: 87.9% -- replaced with /var/log/ceph/cluster1-client.admin.52769.log.gz 2026-02-22T22:12:58.096 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.55847.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48927.log 2026-02-22T22:12:58.097 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.73888.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68940.log 2026-02-22T22:12:58.097 INFO:teuthology.orchestra.run.trial026.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.73888.log.gz 2026-02-22T22:12:58.097 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.48927.log: gzip 3.5% -- replaced with /var/log/ceph/cluster2-client.mirror.48927.log.gz 2026-02-22T22:12:58.097 INFO:teuthology.orchestra.run.trial026.stderr: -5 --verbose -- /var/log/ceph/cluster1-client.mirror.83791.log 2026-02-22T22:12:58.097 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.68940.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66309.log 2026-02-22T22:12:58.098 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.83791.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.73168.log 2026-02-22T22:12:58.098 INFO:teuthology.orchestra.run.trial026.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.83791.log.gz 2026-02-22T22:12:58.098 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.66309.log: 95.0% -- replaced with /var/log/ceph/cluster1-client.mirror.68940.log.gz 2026-02-22T22:12:58.098 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.63597.log 2026-02-22T22:12:58.099 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.73168.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56707.log 2026-02-22T22:12:58.099 INFO:teuthology.orchestra.run.trial026.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.73168.log.gz 2026-02-22T22:12:58.099 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.63597.log: gzip -5 --verbose -- 87.7% /var/log/ceph/cluster2-client.mirror.52475.log 2026-02-22T22:12:58.099 INFO:teuthology.orchestra.run.trial026.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.63597.log.gz 2026-02-22T22:12:58.099 INFO:teuthology.orchestra.run.trial026.stderr: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.66309.log.gz 2026-02-22T22:12:58.099 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.56707.log: 93.4% -- replaced with /var/log/ceph/cluster1-client.mirror.55847.log.gz 2026-02-22T22:12:58.099 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84412.log 2026-02-22T22:12:58.100 INFO:teuthology.orchestra.run.trial026.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.56707.log.gz 2026-02-22T22:12:58.100 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.52475.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75376.log 2026-02-22T22:12:58.100 INFO:teuthology.orchestra.run.trial026.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.52475.log.gz 2026-02-22T22:12:58.100 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.84412.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.63264.log 2026-02-22T22:12:58.100 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.75376.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.48250.log 2026-02-22T22:12:58.100 INFO:teuthology.orchestra.run.trial026.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.84412.log.gz 2026-02-22T22:12:58.101 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.63264.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.72397.log 2026-02-22T22:12:58.101 INFO:teuthology.orchestra.run.trial026.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.75376.log.gz 2026-02-22T22:12:58.101 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.48250.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79128.log 2026-02-22T22:12:58.101 INFO:teuthology.orchestra.run.trial026.stderr: 93.2% -- replaced with /var/log/ceph/cluster1-client.mirror.48250.log.gz 2026-02-22T22:12:58.101 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.72397.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.47547.log 2026-02-22T22:12:58.102 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.79128.log: gzip -5 --verbose -- 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.63264.log.gz 2026-02-22T22:12:58.102 INFO:teuthology.orchestra.run.trial026.stderr: /var/log/ceph/cluster1-client.mirror.52154.log 2026-02-22T22:12:58.102 INFO:teuthology.orchestra.run.trial026.stderr: 86.5% -- replaced with /var/log/ceph/cluster1-client.mirror.79128.log.gz 2026-02-22T22:12:58.102 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.admin.47547.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.47547.log.gz 2026-02-22T22:12:58.102 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56241.log 2026-02-22T22:12:58.102 INFO:teuthology.orchestra.run.trial026.stderr: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.72397.log.gz 2026-02-22T22:12:58.102 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.52154.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70986.log 2026-02-22T22:12:58.103 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.56241.log: gzip -5 88.4% --verbose -- -- replaced with /var/log/ceph/cluster1-client.mirror.52154.log.gz 2026-02-22T22:12:58.103 INFO:teuthology.orchestra.run.trial026.stderr: /var/log/ceph/cluster2-client.mirror.76121.log 2026-02-22T22:12:58.103 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.70986.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49012.log 2026-02-22T22:12:58.104 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.76121.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69171.log 2026-02-22T22:12:58.104 INFO:teuthology.orchestra.run.trial026.stderr: 88.0% 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.70986.log.gz 2026-02-22T22:12:58.104 INFO:teuthology.orchestra.run.trial026.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.76121.log.gz 2026-02-22T22:12:58.104 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.49012.log: 57.7% -- replaced with /var/log/ceph/cluster2-client.mirror.49012.log.gz 2026-02-22T22:12:58.104 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50619.log 2026-02-22T22:12:58.104 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.69171.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.80903.log 2026-02-22T22:12:58.104 INFO:teuthology.orchestra.run.trial026.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.69171.log.gz 2026-02-22T22:12:58.105 INFO:teuthology.orchestra.run.trial026.stderr:gzip/var/log/ceph/cluster1-client.mirror.50619.log: -5 --verbose -- /var/log/ceph/cluster2-client.admin.85583.log 2026-02-22T22:12:58.105 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.80903.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.50619.log.gz 2026-02-22T22:12:58.105 INFO:teuthology.orchestra.run.trial026.stderr:gzip 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.80903.log.gz 2026-02-22T22:12:58.105 INFO:teuthology.orchestra.run.trial026.stderr: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57281.log 2026-02-22T22:12:58.105 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.admin.85583.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69689.log 2026-02-22T22:12:58.105 INFO:teuthology.orchestra.run.trial026.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.85583.log.gz 2026-02-22T22:12:58.106 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.57281.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59524.log 2026-02-22T22:12:58.106 INFO:teuthology.orchestra.run.trial026.stderr: 93.6% -- replaced with /var/log/ceph/cluster2-client.mirror.56241.log.gz/var/log/ceph/cluster1-client.mirror.69689.log: 2026-02-22T22:12:58.106 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.69689.log.gz 2026-02-22T22:12:58.106 INFO:teuthology.orchestra.run.trial026.stderr: /var/log/ceph/cluster1-client.mirror.71472.log 89.6% 2026-02-22T22:12:58.106 INFO:teuthology.orchestra.run.trial026.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.57281.log.gz 2026-02-22T22:12:58.106 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.59524.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.63822.log 2026-02-22T22:12:58.107 INFO:teuthology.orchestra.run.trial026.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.59524.log.gz/var/log/ceph/cluster1-client.mirror.71472.log: 2026-02-22T22:12:58.107 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57758.log 2026-02-22T22:12:58.107 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.63822.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68703.log 2026-02-22T22:12:58.108 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.57758.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.63822.log.gz 2026-02-22T22:12:58.108 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76017.log 2026-02-22T22:12:58.108 INFO:teuthology.orchestra.run.trial026.stderr: 94.8% -- replaced with /var/log/ceph/cluster1-client.mirror.71472.log.gz 2026-02-22T22:12:58.108 INFO:teuthology.orchestra.run.trial026.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.57758.log.gz 2026-02-22T22:12:58.108 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.68703.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79218.log 2026-02-22T22:12:58.108 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.76017.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.83699.log 2026-02-22T22:12:58.108 INFO:teuthology.orchestra.run.trial026.stderr: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.68703.log.gz/var/log/ceph/cluster1-client.mirror.79218.log: 2026-02-22T22:12:58.109 INFO:teuthology.orchestra.run.trial026.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.76017.log.gz 2026-02-22T22:12:58.109 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.79218.log.gz 2026-02-22T22:12:58.109 INFO:teuthology.orchestra.run.trial026.stderr: /var/log/ceph/cluster2-client.mirror.61173.log 2026-02-22T22:12:58.109 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.83699.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75913.log 2026-02-22T22:12:58.109 INFO:teuthology.orchestra.run.trial026.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.83699.log.gz 2026-02-22T22:12:58.109 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.61173.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53503.log 2026-02-22T22:12:58.109 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.75913.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75686.log 2026-02-22T22:12:58.110 INFO:teuthology.orchestra.run.trial026.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.61173.log.gz 2026-02-22T22:12:58.110 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.53503.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75562.log 2026-02-22T22:12:58.110 INFO:teuthology.orchestra.run.trial026.stderr: 88.8% -- replaced with /var/log/ceph/cluster1-client.mirror.53503.log.gz 2026-02-22T22:12:58.110 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.75686.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76903.log 2026-02-22T22:12:58.110 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.75562.log: 88.0%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57948.log 2026-02-22T22:12:58.110 INFO:teuthology.orchestra.run.trial026.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.75686.log.gz 2026-02-22T22:12:58.111 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.76903.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.75562.log.gz 2026-02-22T22:12:58.111 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.75913.log.gz 2026-02-22T22:12:58.111 INFO:teuthology.orchestra.run.trial026.stderr: -- /var/log/ceph/cluster2-client.mirror.58742.log 2026-02-22T22:12:58.111 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.57948.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61215.log 2026-02-22T22:12:58.111 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.58742.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79150.log 2026-02-22T22:12:58.112 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.61215.log: gzip -5 --verbose 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.76903.log.gz 2026-02-22T22:12:58.112 INFO:teuthology.orchestra.run.trial026.stderr: -- /var/log/ceph/cluster2-client.mirror.65885.log 2026-02-22T22:12:58.112 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.79150.log: gzip -5 86.8% --verbose -- /var/log/ceph/cluster1-client.mirror.54700.log 2026-02-22T22:12:58.112 INFO:teuthology.orchestra.run.trial026.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.79150.log.gz 2026-02-22T22:12:58.112 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.65885.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48961.log 2026-02-22T22:12:58.113 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.72529.log 2026-02-22T22:12:58.113 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.54700.log: /var/log/ceph/cluster2-client.mirror.48961.log: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.65885.log.gz 2026-02-22T22:12:58.113 INFO:teuthology.orchestra.run.trial026.stderr: 3.5% -- replaced with /var/log/ceph/cluster2-client.mirror.48961.log.gz 2026-02-22T22:12:58.113 INFO:teuthology.orchestra.run.trial026.stderr: 88.5% -- replaced with /var/log/ceph/cluster1-client.mirror.54700.log.gz 2026-02-22T22:12:58.114 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61609.log 2026-02-22T22:12:58.114 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.72529.log: gzip -5 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.72529.log.gz 2026-02-22T22:12:58.114 INFO:teuthology.orchestra.run.trial026.stderr: --verbose -- /var/log/ceph/cluster2-client.mirror.83379.log 2026-02-22T22:12:58.115 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.61609.log: gzip -5 --verbose 79.9% -- replaced with /var/log/ceph/cluster2-client.mirror.61609.log.gz 2026-02-22T22:12:58.115 INFO:teuthology.orchestra.run.trial026.stderr: -- /var/log/ceph/cluster2-client.mirror.76080.log 2026-02-22T22:12:58.115 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.83379.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60754.log 2026-02-22T22:12:58.115 INFO:teuthology.orchestra.run.trial026.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.83379.log.gz 2026-02-22T22:12:58.116 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.76080.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79398.log 2026-02-22T22:12:58.116 INFO:teuthology.orchestra.run.trial026.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.76080.log.gz 2026-02-22T22:12:58.116 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.60754.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.82663.log 2026-02-22T22:12:58.116 INFO:teuthology.orchestra.run.trial026.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.60754.log.gz 2026-02-22T22:12:58.116 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69436.log 2026-02-22T22:12:58.116 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.79398.log: /var/log/ceph/cluster1-client.mirror.82663.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52523.log 2026-02-22T22:12:58.117 INFO:teuthology.orchestra.run.trial026.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.79398.log.gz 2026-02-22T22:12:58.117 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.69436.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.82663.log.gz 2026-02-22T22:12:58.117 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78453.log 2026-02-22T22:12:58.117 INFO:teuthology.orchestra.run.trial026.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.69436.log.gz 2026-02-22T22:12:58.117 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84286.log 2026-02-22T22:12:58.118 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.52523.log: /var/log/ceph/cluster1-client.mirror.78453.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.72306.log 2026-02-22T22:12:58.118 INFO:teuthology.orchestra.run.trial026.stderr: 88.7% -- replaced with /var/log/ceph/cluster1-client.mirror.52523.log.gz 2026-02-22T22:12:58.118 INFO:teuthology.orchestra.run.trial026.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.78453.log.gz 2026-02-22T22:12:58.118 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.84286.log: 79.6% -- replaced with /var/log/ceph/cluster2-client.mirror.84286.log.gzgzip 2026-02-22T22:12:58.118 INFO:teuthology.orchestra.run.trial026.stderr: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82684.log 2026-02-22T22:12:58.118 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77833.log 2026-02-22T22:12:58.119 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.72306.log: /var/log/ceph/cluster2-client.mirror.82684.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.63127.log 2026-02-22T22:12:58.119 INFO:teuthology.orchestra.run.trial026.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.72306.log.gz 2026-02-22T22:12:58.119 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.77833.log: 88.6%gzip -- replaced with /var/log/ceph/cluster2-client.mirror.82684.log.gz -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56019.log 2026-02-22T22:12:58.119 INFO:teuthology.orchestra.run.trial026.stderr: 2026-02-22T22:12:58.120 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.63127.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80813.log 2026-02-22T22:12:58.120 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.56019.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.63127.log.gz 2026-02-22T22:12:58.120 INFO:teuthology.orchestra.run.trial026.stderr: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.77833.log.gzgzip 2026-02-22T22:12:58.120 INFO:teuthology.orchestra.run.trial026.stderr: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63798.log 2026-02-22T22:12:58.121 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.80813.log: 79.9% -- replaced with /var/log/ceph/cluster2-client.mirror.80813.log.gz 2026-02-22T22:12:58.121 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75603.log 2026-02-22T22:12:58.121 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78160.log 2026-02-22T22:12:58.122 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.63798.log: /var/log/ceph/cluster2-client.mirror.75603.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.63798.log.gz 2026-02-22T22:12:58.122 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.54884.log 2026-02-22T22:12:58.122 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.78160.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.78160.log.gz 2026-02-22T22:12:58.123 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.46915.log 2026-02-22T22:12:58.123 INFO:teuthology.orchestra.run.trial026.stderr: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.56019.log.gz 2026-02-22T22:12:58.123 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.admin.54884.log: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.75603.log.gz 2026-02-22T22:12:58.124 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53217.log 2026-02-22T22:12:58.124 INFO:teuthology.orchestra.run.trial026.stderr: 88.3% -- replaced with /var/log/ceph/cluster1-client.admin.54884.log.gz/var/log/ceph/cluster2-client.admin.46915.log: 2026-02-22T22:12:58.124 INFO:teuthology.orchestra.run.trial026.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.46915.log.gz 2026-02-22T22:12:58.124 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.80277.log 2026-02-22T22:12:58.124 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.53217.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67361.log 2026-02-22T22:12:58.125 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.80277.log: 2.6% -- replaced with /var/log/ceph/cluster1-client.mirror.80277.log.gzgzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54340.log 2026-02-22T22:12:58.125 INFO:teuthology.orchestra.run.trial026.stderr: 2026-02-22T22:12:58.125 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84498.log 2026-02-22T22:12:58.125 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.67361.log: /var/log/ceph/cluster2-client.mirror.54340.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.67361.log.gz 2026-02-22T22:12:58.126 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57353.log 2026-02-22T22:12:58.126 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.84498.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.84498.log.gz 2026-02-22T22:12:58.127 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77048.log 2026-02-22T22:12:58.127 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.57353.log: 79.7% -- replaced with /var/log/ceph/cluster2-client.mirror.57353.log.gz 2026-02-22T22:12:58.127 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67108.log 2026-02-22T22:12:58.128 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.77048.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.77048.log.gz 2026-02-22T22:12:58.128 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57906.log 2026-02-22T22:12:58.129 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.67108.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.67108.log.gz 2026-02-22T22:12:58.129 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.48704.log 2026-02-22T22:12:58.130 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.57906.log: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.57906.log.gz 2026-02-22T22:12:58.130 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53663.log 2026-02-22T22:12:58.130 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.48704.log: 3.5% -- replaced with /var/log/ceph/cluster1-client.mirror.48704.log.gz 2026-02-22T22:12:58.131 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53352.log 2026-02-22T22:12:58.134 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.53663.log: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.53663.log.gz 2026-02-22T22:12:58.135 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60880.log 2026-02-22T22:12:58.135 INFO:teuthology.orchestra.run.trial026.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.61261.log.gz 2026-02-22T22:12:58.135 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.53352.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.53352.log.gz 2026-02-22T22:12:58.135 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60816.log 2026-02-22T22:12:58.136 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.60880.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54676.log 2026-02-22T22:12:58.136 INFO:teuthology.orchestra.run.trial026.stderr: 79.8% -- replaced with /var/log/ceph/cluster2-client.mirror.60880.log.gz 2026-02-22T22:12:58.136 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.60816.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76614.log 2026-02-22T22:12:58.137 INFO:teuthology.orchestra.run.trial026.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.60816.log.gz 2026-02-22T22:12:58.137 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.54676.log: 89.2% -- replaced with /var/log/ceph/cluster1-client.mirror.54676.log.gz 2026-02-22T22:12:58.137 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60836.log 2026-02-22T22:12:58.138 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69617.log 2026-02-22T22:12:58.138 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.76614.log: /var/log/ceph/cluster2-client.mirror.60836.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.76614.log.gz 2026-02-22T22:12:58.138 INFO:teuthology.orchestra.run.trial026.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.60836.log.gz 2026-02-22T22:12:58.139 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60267.log 2026-02-22T22:12:58.139 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-osd.0.log 2026-02-22T22:12:58.139 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.69617.log: /var/log/ceph/cluster2-client.mirror.60267.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.69617.log.gz 2026-02-22T22:12:58.139 INFO:teuthology.orchestra.run.trial026.stderr: 79.4% -- replaced with /var/log/ceph/cluster2-client.mirror.60267.log.gz 2026-02-22T22:12:58.139 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84518.log 2026-02-22T22:12:58.140 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59971.log 2026-02-22T22:12:58.142 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-osd.0.log: /var/log/ceph/cluster2-client.mirror.84518.log: 91.6% -- replaced with /var/log/ceph/cluster2-client.mirror.84518.log.gz 2026-02-22T22:12:58.143 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58784.log 2026-02-22T22:12:58.148 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.59971.log: 94.1% -- replaced with /var/log/ceph/cluster2-client.mirror.60370.log.gz 2026-02-22T22:12:58.148 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75273.log 2026-02-22T22:12:58.149 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.58784.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.58784.log.gz 2026-02-22T22:12:58.150 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65700.log 2026-02-22T22:12:58.150 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.75273.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.75273.log.gz 2026-02-22T22:12:58.150 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59607.log 2026-02-22T22:12:58.151 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.65700.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.65700.log.gz 2026-02-22T22:12:58.152 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66019.log 2026-02-22T22:12:58.152 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.59607.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.59607.log.gz 2026-02-22T22:12:58.152 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72927.log 2026-02-22T22:12:58.153 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.66019.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.66019.log.gz 2026-02-22T22:12:58.153 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67606.log 2026-02-22T22:12:58.154 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.72927.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.72927.log.gz 2026-02-22T22:12:58.154 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.64111.log 2026-02-22T22:12:58.154 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.67606.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.67606.log.gz 2026-02-22T22:12:58.155 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58848.log 2026-02-22T22:12:58.155 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.64111.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.64111.log.gz 2026-02-22T22:12:58.156 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60505.log 2026-02-22T22:12:58.156 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.58848.log: 79.8% -- replaced with /var/log/ceph/cluster2-client.mirror.58848.log.gz 2026-02-22T22:12:58.156 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57780.log 2026-02-22T22:12:58.157 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.60505.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.60505.log.gz 2026-02-22T22:12:58.157 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55113.log 2026-02-22T22:12:58.179 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.57780.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57004.log 2026-02-22T22:12:58.188 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.55113.log: 93.5%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62691.log 2026-02-22T22:12:58.195 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.57004.log: 94.9% -- replaced with /var/log/ceph/cluster2-client.mirror.55113.log.gz 2026-02-22T22:12:58.202 INFO:teuthology.orchestra.run.trial026.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.57004.log.gz 2026-02-22T22:12:58.208 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81997.log 2026-02-22T22:12:58.209 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.62691.log: 90.0% -- replaced with /var/log/ceph/cluster2-client.mirror.62691.log.gz 2026-02-22T22:12:58.217 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55020.log 2026-02-22T22:12:58.227 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.81997.log: 88.1%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62240.log 2026-02-22T22:12:58.230 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.55020.log: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.81997.log.gz 2026-02-22T22:12:58.236 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51204.log 2026-02-22T22:12:58.239 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.62240.log: 89.7% -- replaced with /var/log/ceph/cluster1-client.mirror.55020.log.gz 2026-02-22T22:12:58.245 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75831.log 2026-02-22T22:12:58.250 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.51204.log: 93.3% -- replaced with /var/log/ceph/cluster2-client.mirror.62240.log.gz 2026-02-22T22:12:58.256 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57370.log 2026-02-22T22:12:58.257 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.75831.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.75831.log.gz 2026-02-22T22:12:58.261 INFO:teuthology.orchestra.run.trial026.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.51204.log.gz 2026-02-22T22:12:58.267 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70154.log 2026-02-22T22:12:58.268 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.57370.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.57370.log.gz 2026-02-22T22:12:58.277 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61133.log 2026-02-22T22:12:58.286 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.70154.log: 87.8%gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.64493.log 2026-02-22T22:12:58.289 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.61133.log: -- replaced with /var/log/ceph/cluster1-client.mirror.70154.log.gz 2026-02-22T22:12:58.295 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77854.log 2026-02-22T22:12:58.309 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.64493.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.64493.log.gz 2026-02-22T22:12:58.309 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58633.log 2026-02-22T22:12:58.309 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.77854.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.77854.log.gz 2026-02-22T22:12:58.322 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56105.log 2026-02-22T22:12:58.323 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.58633.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.58633.log.gz 2026-02-22T22:12:58.332 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56627.log 2026-02-22T22:12:58.345 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.56105.log: 93.5%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.2.60983.log 2026-02-22T22:12:58.345 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.56627.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.56627.log.gz 2026-02-22T22:12:58.346 INFO:teuthology.orchestra.run.trial026.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.59099.log.gz 2026-02-22T22:12:58.349 INFO:teuthology.orchestra.run.trial026.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.56105.log.gz 2026-02-22T22:12:58.352 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.62399.log 2026-02-22T22:12:58.362 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.2.60983.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.64288.log 2026-02-22T22:12:58.371 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.62399.log: 88.0%gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78273.log 2026-02-22T22:12:58.374 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.64288.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.62399.log.gz 2026-02-22T22:12:58.380 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77916.log 2026-02-22T22:12:58.383 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.78273.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.64288.log.gz 2026-02-22T22:12:58.392 INFO:teuthology.orchestra.run.trial026.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.78273.log.gz 2026-02-22T22:12:58.394 INFO:teuthology.orchestra.run.trial026.stderr: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.2.60983.log.gz 2026-02-22T22:12:58.400 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51111.log 2026-02-22T22:12:58.401 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.77916.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.77916.log.gz 2026-02-22T22:12:58.416 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55374.log 2026-02-22T22:12:58.416 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.51111.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.51111.log.gz 2026-02-22T22:12:58.428 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70918.log 2026-02-22T22:12:58.432 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.55374.log: 93.3% -- replaced with /var/log/ceph/cluster1-client.mirror.55374.log.gz 2026-02-22T22:12:58.442 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55354.log 2026-02-22T22:12:58.443 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.70918.log: 92.5% -- replaced with /var/log/ceph/cluster1-client.mirror.70918.log.gz 2026-02-22T22:12:58.458 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70273.log 2026-02-22T22:12:58.462 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.55354.log: 93.4% -- replaced with /var/log/ceph/cluster2-client.mirror.55354.log.gz 2026-02-22T22:12:58.472 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79713.log 2026-02-22T22:12:58.472 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.70273.log: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.70273.log.gz 2026-02-22T22:12:58.483 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82018.log 2026-02-22T22:12:58.484 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.79713.log: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.79713.log.gz 2026-02-22T22:12:58.497 INFO:teuthology.orchestra.run.trial004.stderr: 90.3% -- replaced with /var/log/ceph/cluster1-mon.a.log.gz 2026-02-22T22:12:58.498 INFO:teuthology.orchestra.run.trial026.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.58907.log.gz 2026-02-22T22:12:58.500 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77172.log 2026-02-22T22:12:58.500 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.82018.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.82018.log.gz 2026-02-22T22:12:58.500 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60304.log 2026-02-22T22:12:58.502 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.77172.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.77172.log.gz 2026-02-22T22:12:58.502 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.72618.log 2026-02-22T22:12:58.502 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.60304.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.60304.log.gz 2026-02-22T22:12:58.503 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61074.log 2026-02-22T22:12:58.505 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.72618.log: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.72618.log.gz 2026-02-22T22:12:58.505 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65128.log 2026-02-22T22:12:58.505 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.61074.log: 79.7% -- replaced with /var/log/ceph/cluster2-client.mirror.61074.log.gz 2026-02-22T22:12:58.505 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77110.log 2026-02-22T22:12:58.515 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.65128.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.65128.log.gz 2026-02-22T22:12:58.515 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.71108.log 2026-02-22T22:12:58.515 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.77110.log: 93.9% -- replaced with /var/log/ceph/cluster2-client.mirror.80371.log.gz 2026-02-22T22:12:58.515 INFO:teuthology.orchestra.run.trial026.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.77110.log.gz 2026-02-22T22:12:58.515 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76449.log 2026-02-22T22:12:58.515 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.71108.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.77983.log 2026-02-22T22:12:58.515 INFO:teuthology.orchestra.run.trial026.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.71108.log.gz 2026-02-22T22:12:58.516 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.76449.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58322.log 2026-02-22T22:12:58.516 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.77983.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.76449.log.gz 2026-02-22T22:12:58.516 INFO:teuthology.orchestra.run.trial026.stderr: 2.6% -- replaced with /var/log/ceph/cluster1-client.mirror.77983.log.gz 2026-02-22T22:12:58.516 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81789.log 2026-02-22T22:12:58.516 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.58322.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67981.log 2026-02-22T22:12:58.516 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.81789.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.81789.log.gz 2026-02-22T22:12:58.516 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/ceph.tmp-client.admin.38384.log 2026-02-22T22:12:58.516 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.67981.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.67981.log.gz 2026-02-22T22:12:58.516 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67727.log 2026-02-22T22:12:58.516 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/ceph.tmp-client.admin.38384.log: 0.0% -- replaced with /var/log/ceph/ceph.tmp-client.admin.38384.log.gz 2026-02-22T22:12:58.516 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.62628.log 2026-02-22T22:12:58.516 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.67727.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.67727.log.gz 2026-02-22T22:12:58.517 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73118.log 2026-02-22T22:12:58.517 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.62628.log: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.62628.log.gz 2026-02-22T22:12:58.517 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68623.log 2026-02-22T22:12:58.517 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.73118.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.73118.log.gz 2026-02-22T22:12:58.517 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76142.log 2026-02-22T22:12:58.521 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.68623.log: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.68623.log.gz 2026-02-22T22:12:58.521 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76841.log 2026-02-22T22:12:58.521 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.76142.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.76142.log.gz 2026-02-22T22:12:58.521 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63103.log 2026-02-22T22:12:58.521 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.76841.log: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.76841.log.gz 2026-02-22T22:12:58.521 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55632.log 2026-02-22T22:12:58.521 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.63103.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.63103.log.gz 2026-02-22T22:12:58.521 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78903.log 2026-02-22T22:12:58.525 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.55632.log: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.55632.log.gz 2026-02-22T22:12:58.526 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57151.log 2026-02-22T22:12:58.526 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.78903.log: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.78903.log.gz 2026-02-22T22:12:58.526 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52243.log 2026-02-22T22:12:58.534 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.57151.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55198.log 2026-02-22T22:12:58.544 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.52243.log: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.59890.log.gz 2026-02-22T22:12:58.549 INFO:teuthology.orchestra.run.trial026.stderr: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.52243.log.gz 2026-02-22T22:12:58.549 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54764.log 2026-02-22T22:12:58.550 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.55198.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.55198.log.gz 2026-02-22T22:12:58.550 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65774.log 2026-02-22T22:12:58.555 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.54764.log: 93.6% -- replaced with /var/log/ceph/cluster2-client.mirror.54764.log.gz 2026-02-22T22:12:58.556 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72823.log 2026-02-22T22:12:58.556 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.65774.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.65774.log.gz 2026-02-22T22:12:58.556 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.62833.log 2026-02-22T22:12:58.558 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.72823.log: 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.72823.log.gz 2026-02-22T22:12:58.559 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.64855.log 2026-02-22T22:12:58.559 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.62833.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.62833.log.gz 2026-02-22T22:12:58.559 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57738.log 2026-02-22T22:12:58.560 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.64855.log: 92.2% -- replaced with /var/log/ceph/cluster1-client.mirror.64855.log.gz 2026-02-22T22:12:58.560 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69362.log 2026-02-22T22:12:58.563 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.57738.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.57738.log.gz 2026-02-22T22:12:58.563 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66592.log 2026-02-22T22:12:58.563 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.69362.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.69362.log.gz 2026-02-22T22:12:58.563 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76059.log 2026-02-22T22:12:58.565 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.66592.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.66592.log.gz 2026-02-22T22:12:58.565 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68447.log 2026-02-22T22:12:58.565 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.76059.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.76059.log.gz 2026-02-22T22:12:58.565 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.73023.log 2026-02-22T22:12:58.567 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.68447.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.68447.log.gz 2026-02-22T22:12:58.568 INFO:teuthology.orchestra.run.trial026.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.59971.log.gz 2026-02-22T22:12:58.568 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.76268.log 2026-02-22T22:12:58.568 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.73023.log: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.73023.log.gz 2026-02-22T22:12:58.568 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61420.log 2026-02-22T22:12:58.568 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.1.76268.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76821.log 2026-02-22T22:12:58.575 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.61420.log: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.60527.log.gz 2026-02-22T22:12:58.579 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57545.log 2026-02-22T22:12:58.579 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.76821.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.76821.log.gz 2026-02-22T22:12:58.579 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67132.log 2026-02-22T22:12:58.579 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.57545.log: 91.7% 79.7% -- replaced with /var/log/ceph/cluster2-client.mirror.57545.log.gz 2026-02-22T22:12:58.579 INFO:teuthology.orchestra.run.trial026.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.1.76268.log.gz 2026-02-22T22:12:58.579 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79668.log 2026-02-22T22:12:58.579 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.67132.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61748.log 2026-02-22T22:12:58.580 INFO:teuthology.orchestra.run.trial026.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.67132.log.gz 2026-02-22T22:12:58.580 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.79668.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49105.log 2026-02-22T22:12:58.580 INFO:teuthology.orchestra.run.trial026.stderr: 86.7%/var/log/ceph/cluster2-client.mirror.61748.log: -- replaced with /var/log/ceph/cluster1-client.mirror.79668.log.gz 2026-02-22T22:12:58.588 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75293.log 2026-02-22T22:12:58.588 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.49105.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.49105.log.gz 2026-02-22T22:12:58.588 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62303.log 2026-02-22T22:12:58.590 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.75293.log: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.75293.log.gz 2026-02-22T22:12:58.590 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76986.log 2026-02-22T22:12:58.591 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.62303.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.62303.log.gz 2026-02-22T22:12:58.592 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56447.log 2026-02-22T22:12:58.592 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.76986.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.76986.log.gz 2026-02-22T22:12:58.593 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.73216.log 2026-02-22T22:12:58.593 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.56447.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.56447.log.gz 2026-02-22T22:12:58.594 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77875.log 2026-02-22T22:12:58.594 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.73216.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.73216.log.gz 2026-02-22T22:12:58.595 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73975.log 2026-02-22T22:12:58.595 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.77875.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.77875.log.gz 2026-02-22T22:12:58.596 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.62898.log 2026-02-22T22:12:58.597 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.73975.log: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.73975.log.gz 2026-02-22T22:12:58.597 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82517.log 2026-02-22T22:12:58.598 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.62898.log: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.62898.log.gz 2026-02-22T22:12:58.599 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60326.log 2026-02-22T22:12:58.599 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.82517.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.82517.log.gz 2026-02-22T22:12:58.600 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.83293.log 2026-02-22T22:12:58.609 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.60326.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61378.log 2026-02-22T22:12:58.610 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.83293.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.83293.log.gz 2026-02-22T22:12:58.625 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69842.log 2026-02-22T22:12:58.626 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.61378.log: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.61378.log.gz 2026-02-22T22:12:58.632 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69308.log 2026-02-22T22:12:58.645 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.69842.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.69842.log.gz 2026-02-22T22:12:58.647 INFO:teuthology.orchestra.run.trial026.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.59374.log.gz 2026-02-22T22:12:58.648 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48995.log 2026-02-22T22:12:58.650 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.69308.log: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.69308.log.gz 2026-02-22T22:12:58.650 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67289.log 2026-02-22T22:12:58.651 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.48995.log: 57.7% -- replaced with /var/log/ceph/cluster2-client.mirror.48995.log.gz 2026-02-22T22:12:58.651 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80769.log 2026-02-22T22:12:58.652 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.67289.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.67289.log.gz 2026-02-22T22:12:58.652 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.62965.log 2026-02-22T22:12:58.653 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.80769.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.80769.log.gz 2026-02-22T22:12:58.653 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.64742.log 2026-02-22T22:12:58.655 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.62965.log: 95.0% -- replaced with /var/log/ceph/cluster1-client.mirror.62965.log.gz 2026-02-22T22:12:58.656 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68576.log 2026-02-22T22:12:58.656 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.64742.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.64742.log.gz 2026-02-22T22:12:58.657 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81769.log 2026-02-22T22:12:58.657 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.68576.log: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.68576.log.gz 2026-02-22T22:12:58.658 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.71253.log 2026-02-22T22:12:58.659 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.81769.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.81769.log.gz 2026-02-22T22:12:58.659 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63286.log 2026-02-22T22:12:58.660 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.71253.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.71253.log.gz 2026-02-22T22:12:58.660 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56607.log 2026-02-22T22:12:58.661 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.63286.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.63286.log.gz 2026-02-22T22:12:58.661 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58428.log 2026-02-22T22:12:58.662 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.56607.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.56607.log.gz 2026-02-22T22:12:58.662 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57264.log 2026-02-22T22:12:58.663 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.58428.log: 79.7% -- replaced with /var/log/ceph/cluster2-client.mirror.58428.log.gz 2026-02-22T22:12:58.663 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84345.log 2026-02-22T22:12:58.664 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.57264.log: 79.7% -- replaced with /var/log/ceph/cluster2-client.mirror.57264.log.gz 2026-02-22T22:12:58.664 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.72754.log 2026-02-22T22:12:58.674 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.84345.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63690.log 2026-02-22T22:12:58.686 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.72754.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.72754.log.gz 2026-02-22T22:12:58.692 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77255.log 2026-02-22T22:12:58.694 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.63690.log: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.63690.log.gz 2026-02-22T22:12:58.700 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65215.log 2026-02-22T22:12:58.704 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.77255.log: 88.1% 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.61215.log.gz 2026-02-22T22:12:58.705 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54721.log 2026-02-22T22:12:58.705 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.65215.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.65215.log.gz 2026-02-22T22:12:58.705 INFO:teuthology.orchestra.run.trial026.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.77255.log.gz 2026-02-22T22:12:58.706 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59437.log 2026-02-22T22:12:58.709 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.54721.log: 93.6% -- replaced with /var/log/ceph/cluster2-client.mirror.54721.log.gz 2026-02-22T22:12:58.710 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62107.log 2026-02-22T22:12:58.710 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.59437.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.59437.log.gz 2026-02-22T22:12:58.711 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.82475.log 2026-02-22T22:12:58.720 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.62107.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58655.log 2026-02-22T22:12:58.732 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.82475.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.82475.log.gz 2026-02-22T22:12:58.736 INFO:teuthology.orchestra.run.trial026.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.61133.log.gz 2026-02-22T22:12:58.736 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57582.log 2026-02-22T22:12:58.742 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.58655.log: 93.9% -- replaced with /var/log/ceph/cluster2-client.mirror.49780.log.gz 2026-02-22T22:12:58.743 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84093.log 2026-02-22T22:12:58.743 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.57582.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.57582.log.gz 2026-02-22T22:12:58.744 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56507.log 2026-02-22T22:12:58.754 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.84093.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70458.log 2026-02-22T22:12:58.754 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.56507.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.56507.log.gz 2026-02-22T22:12:58.766 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-osd.1.log 2026-02-22T22:12:58.767 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.70458.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.70458.log.gz 2026-02-22T22:12:58.776 INFO:teuthology.orchestra.run.trial026.stderr: 93.9% -- replaced with /var/log/ceph/cluster2-client.mirror.58571.log.gz 2026-02-22T22:12:58.777 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66220.log 2026-02-22T22:12:58.785 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-osd.1.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60799.log 2026-02-22T22:12:58.798 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.66220.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.66220.log.gz 2026-02-22T22:12:58.807 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.48619.log 2026-02-22T22:12:58.807 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.60799.log: 79.5% -- replaced with /var/log/ceph/cluster2-client.mirror.60799.log.gz 2026-02-22T22:12:58.823 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51770.log 2026-02-22T22:12:58.823 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.admin.48619.log: 80.3% -- replaced with /var/log/ceph/cluster1-client.admin.48619.log.gz 2026-02-22T22:12:58.835 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61877.log 2026-02-22T22:12:58.836 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.51770.log: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.51770.log.gz 2026-02-22T22:12:58.851 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.56150.log 2026-02-22T22:12:58.854 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.61877.log: 95.4% -- replaced with /var/log/ceph/cluster1-client.mirror.61877.log.gz 2026-02-22T22:12:58.865 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65569.log 2026-02-22T22:12:58.881 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.1.56150.log: 93.9% -- replaced with /var/log/ceph/cluster2-client.mirror.82617.log.gz 2026-02-22T22:12:58.881 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58347.log 2026-02-22T22:12:58.881 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.65569.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.65569.log.gz 2026-02-22T22:12:58.881 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66755.log 2026-02-22T22:12:58.881 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.58347.log: 79.7% -- replaced with /var/log/ceph/cluster2-client.mirror.58347.log.gz 2026-02-22T22:12:58.881 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60776.log 2026-02-22T22:12:58.881 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.66755.log: 93.9% -- replaced with /var/log/ceph/cluster2-client.mirror.51923.log.gz 2026-02-22T22:12:58.882 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55397.log 2026-02-22T22:12:58.882 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.60776.log: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.66755.log.gz 2026-02-22T22:12:58.882 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52701.log 2026-02-22T22:12:58.885 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.55397.log: 93.3% -- replaced with /var/log/ceph/cluster2-client.mirror.55397.log.gz 2026-02-22T22:12:58.885 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80294.log 2026-02-22T22:12:58.890 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.52701.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.52701.log.gz 2026-02-22T22:12:58.890 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82322.log 2026-02-22T22:12:58.890 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.80294.log: 1.3% -- replaced with /var/log/ceph/cluster2-client.mirror.80294.log.gz 2026-02-22T22:12:58.891 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.50979.log 2026-02-22T22:12:58.891 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.82322.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.82322.log.gz 2026-02-22T22:12:58.891 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57843.log 2026-02-22T22:12:58.891 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.admin.50979.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.50979.log.gz 2026-02-22T22:12:58.891 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67627.log 2026-02-22T22:12:58.891 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.57843.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.57843.log.gz 2026-02-22T22:12:58.891 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65593.log 2026-02-22T22:12:58.913 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.67627.log: 93.9% -- replaced with /var/log/ceph/cluster2-client.mirror.60692.log.gz 2026-02-22T22:12:58.913 INFO:teuthology.orchestra.run.trial026.stderr: 93.9% -- replaced with /var/log/ceph/cluster2-client.mirror.53217.log.gz 2026-02-22T22:12:58.913 INFO:teuthology.orchestra.run.trial026.stderr: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.67627.log.gz 2026-02-22T22:12:58.913 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68148.log 2026-02-22T22:12:58.913 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose/var/log/ceph/cluster1-client.mirror.65593.log: -- /var/log/ceph/cluster1-client.mirror.52499.log 2026-02-22T22:12:58.913 INFO:teuthology.orchestra.run.trial026.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.65593.log.gz 2026-02-22T22:12:58.913 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.68148.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66859.log 2026-02-22T22:12:58.913 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.52499.log: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.68148.log.gz 88.5% 2026-02-22T22:12:58.914 INFO:teuthology.orchestra.run.trial026.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.52499.log.gz 2026-02-22T22:12:58.914 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70371.log 2026-02-22T22:12:58.914 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose/var/log/ceph/cluster1-client.mirror.66859.log: -- /var/log/ceph/cluster1-client.mirror.65545.log 2026-02-22T22:12:58.914 INFO:teuthology.orchestra.run.trial026.stderr: 87.7%/var/log/ceph/cluster1-client.mirror.70371.log: -- replaced with /var/log/ceph/cluster1-client.mirror.66859.log.gz 2026-02-22T22:12:58.914 INFO:teuthology.orchestra.run.trial026.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.70371.log.gz 2026-02-22T22:12:58.914 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84685.log 2026-02-22T22:12:58.914 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.65545.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63547.log 2026-02-22T22:12:58.914 INFO:teuthology.orchestra.run.trial026.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.65545.log.gz 2026-02-22T22:12:58.914 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.84685.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.mirror.84685.log.gz 2026-02-22T22:12:58.914 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66931.log 2026-02-22T22:12:58.914 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.63547.log: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.63547.log.gz 2026-02-22T22:12:58.914 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76531.log 2026-02-22T22:12:58.914 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.66931.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79735.log 2026-02-22T22:12:58.914 INFO:teuthology.orchestra.run.trial026.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.66931.log.gz 2026-02-22T22:12:58.914 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.76531.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67558.log 2026-02-22T22:12:58.915 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.79735.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.79735.log.gz 2026-02-22T22:12:58.915 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.73912.log 2026-02-22T22:12:58.915 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.67558.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.67558.log.gz 2026-02-22T22:12:58.915 INFO:teuthology.orchestra.run.trial026.stderr: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.76531.log.gz 2026-02-22T22:12:58.915 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67177.log 2026-02-22T22:12:58.915 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.73912.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66196.log 2026-02-22T22:12:58.915 INFO:teuthology.orchestra.run.trial026.stderr: 88.0% -- replaced with /var/log/ceph/cluster1-client.mirror.73912.log.gz 2026-02-22T22:12:58.915 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.67177.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56893.log 2026-02-22T22:12:58.915 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.66196.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.66196.log.gz 2026-02-22T22:12:58.915 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.67177.log.gz 2026-02-22T22:12:58.915 INFO:teuthology.orchestra.run.trial026.stderr: /var/log/ceph/cluster2-client.mirror.64718.log 2026-02-22T22:12:58.915 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.56893.log: 79.7% -- replaced with /var/log/ceph/cluster2-client.mirror.56893.log.gz 2026-02-22T22:12:58.915 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68094.log 2026-02-22T22:12:58.915 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.64718.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56547.log 2026-02-22T22:12:58.915 INFO:teuthology.orchestra.run.trial026.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.64718.log.gz 2026-02-22T22:12:58.916 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.68094.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.46553.log 2026-02-22T22:12:58.916 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.56547.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.56547.log.gz 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.68094.log.gz 2026-02-22T22:12:58.916 INFO:teuthology.orchestra.run.trial026.stderr: 2026-02-22T22:12:58.916 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.80005.log 2026-02-22T22:12:58.916 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.admin.46553.log: 0.0%gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70877.log 2026-02-22T22:12:58.916 INFO:teuthology.orchestra.run.trial026.stderr: -- replaced with /var/log/ceph/cluster2-client.admin.46553.log.gz 2026-02-22T22:12:58.916 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77565.log 2026-02-22T22:12:58.916 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.80005.log: /var/log/ceph/cluster1-client.mirror.70877.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.80005.log.gz 2026-02-22T22:12:58.916 INFO:teuthology.orchestra.run.trial026.stderr: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.70877.log.gz 2026-02-22T22:12:58.916 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.73667.log 2026-02-22T22:12:58.916 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.51971.log 2026-02-22T22:12:58.916 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.77565.log: /var/log/ceph/cluster1-client.mirror.73667.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.77565.log.gz 2026-02-22T22:12:58.916 INFO:teuthology.orchestra.run.trial026.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.73667.log.gz 2026-02-22T22:12:58.916 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50643.log 2026-02-22T22:12:58.916 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.admin.51971.log: gzip -5 --verbose 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.51971.log.gz 2026-02-22T22:12:58.917 INFO:teuthology.orchestra.run.trial026.stderr: -- /var/log/ceph/cluster2-client.mirror.67484.log 2026-02-22T22:12:58.917 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79173.log 2026-02-22T22:12:58.917 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.50643.log: /var/log/ceph/cluster2-client.mirror.67484.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.50643.log.gz 2026-02-22T22:12:58.917 INFO:teuthology.orchestra.run.trial026.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.67484.log.gz 2026-02-22T22:12:58.917 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.51458.log 2026-02-22T22:12:58.917 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.79173.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.79173.log.gz 2026-02-22T22:12:58.917 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67864.log 2026-02-22T22:12:58.917 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.2.51458.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56728.log 2026-02-22T22:12:58.917 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.67864.log: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.67864.log.gz 2026-02-22T22:12:58.920 INFO:teuthology.orchestra.run.trial026.stderr: 93.0% -- replaced with /var/log/ceph/cluster1-client.mirror.2.51458.log.gz 2026-02-22T22:12:58.920 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68519.log 2026-02-22T22:12:58.920 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.56728.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.56728.log.gz 2026-02-22T22:12:58.920 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51722.log 2026-02-22T22:12:58.920 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.3.81032.log 2026-02-22T22:12:58.920 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.68519.log: /var/log/ceph/cluster1-client.mirror.51722.log: 88.2% -- replaced with /var/log/ceph/cluster1-client.mirror.51722.log.gz 2026-02-22T22:12:58.920 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75707.log 2026-02-22T22:12:58.921 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.3.81032.log: 93.0% -- replaced with /var/log/ceph/cluster2-client.mirror.3.81032.log.gz 2026-02-22T22:12:58.921 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81888.log 2026-02-22T22:12:58.922 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.75707.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.75707.log.gz 2026-02-22T22:12:58.922 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70416.log 2026-02-22T22:12:58.923 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.81888.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.81888.log.gz 2026-02-22T22:12:58.923 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56647.log 2026-02-22T22:12:58.925 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.70416.log: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.70416.log.gz 2026-02-22T22:12:58.925 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56304.log 2026-02-22T22:12:58.926 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.56647.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.56647.log.gz 2026-02-22T22:12:58.928 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56930.log 2026-02-22T22:12:58.928 INFO:teuthology.orchestra.run.trial026.stderr: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.68519.log.gz 2026-02-22T22:12:58.928 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.56304.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.74302.log 2026-02-22T22:12:58.928 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.56930.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.56930.log.gz 2026-02-22T22:12:58.928 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.71860.log 2026-02-22T22:12:58.928 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.74302.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.74302.log.gz 2026-02-22T22:12:58.930 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58683.log 2026-02-22T22:12:58.930 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.71860.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.71860.log.gz 2026-02-22T22:12:58.930 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.48093.log 2026-02-22T22:12:58.930 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.58683.log: 79.8% -- replaced with /var/log/ceph/cluster2-client.mirror.58683.log.gz 2026-02-22T22:12:58.934 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66642.log 2026-02-22T22:12:58.934 INFO:teuthology.orchestra.run.trial026.stderr: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.56304.log.gz 2026-02-22T22:12:58.935 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.2.48093.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.47890.log 2026-02-22T22:12:58.935 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.66642.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.66642.log.gz 2026-02-22T22:12:58.935 INFO:teuthology.orchestra.run.trial026.stderr: 93.2% -- replaced with /var/log/ceph/cluster1-client.mirror.1.56150.log.gz 2026-02-22T22:12:58.935 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57177.log 2026-02-22T22:12:58.935 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.admin.47890.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.47890.log.gz 2026-02-22T22:12:58.935 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76676.log 2026-02-22T22:12:58.935 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5/var/log/ceph/cluster2-client.mirror.57177.log: --verbose -- /var/log/ceph/cluster1-client.mirror.69890.log 2026-02-22T22:12:58.935 INFO:teuthology.orchestra.run.trial026.stderr: 79.7% -- replaced with /var/log/ceph/cluster2-client.mirror.57177.log.gz 2026-02-22T22:12:58.935 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.76676.log: 92.9% -- replaced with /var/log/ceph/cluster1-client.mirror.2.48093.log.gz 2026-02-22T22:12:58.935 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55612.log 2026-02-22T22:12:58.935 INFO:teuthology.orchestra.run.trial026.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.76676.log.gz 2026-02-22T22:12:58.935 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.69890.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59459.log 2026-02-22T22:12:58.935 INFO:teuthology.orchestra.run.trial026.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.69890.log.gz/var/log/ceph/cluster2-client.mirror.55612.log: 2026-02-22T22:12:58.935 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59992.log 2026-02-22T22:12:58.935 INFO:teuthology.orchestra.run.trial026.stderr:gzip/var/log/ceph/cluster2-client.mirror.59459.log: -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55718.log 2026-02-22T22:12:58.936 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.59992.log: 79.8% -- replaced with /var/log/ceph/cluster2-client.mirror.59992.log.gz 2026-02-22T22:12:58.936 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60717.log 2026-02-22T22:12:58.937 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.55718.log: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.55612.log.gz 2026-02-22T22:12:58.938 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77585.log 2026-02-22T22:12:58.938 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.60717.log: 79.5% -- replaced with /var/log/ceph/cluster2-client.mirror.60717.log.gz 2026-02-22T22:12:58.938 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59077.log 2026-02-22T22:12:58.939 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.77585.log: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.55718.log.gz 2026-02-22T22:12:58.939 INFO:teuthology.orchestra.run.trial026.stderr: 90.4%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62506.log 2026-02-22T22:12:58.940 INFO:teuthology.orchestra.run.trial026.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.77585.log.gz 2026-02-22T22:12:58.941 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.59077.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76883.log 2026-02-22T22:12:58.941 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.62506.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.59077.log.gz 2026-02-22T22:12:58.942 INFO:teuthology.orchestra.run.trial026.stderr:gzip 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.62506.log.gz 2026-02-22T22:12:58.942 INFO:teuthology.orchestra.run.trial026.stderr: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71388.log 2026-02-22T22:12:58.942 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.0.83837.log 2026-02-22T22:12:58.942 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.76883.log: /var/log/ceph/cluster2-client.mirror.71388.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.76883.log.gz 2026-02-22T22:12:58.943 INFO:teuthology.orchestra.run.trial026.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.71388.log.gz 2026-02-22T22:12:58.943 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65634.log 2026-02-22T22:12:58.943 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53089.log 2026-02-22T22:12:58.946 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.0.83837.log: /var/log/ceph/cluster1-client.mirror.65634.log: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.65634.log.gz 2026-02-22T22:12:58.946 INFO:teuthology.orchestra.run.trial026.stderr: 92.3% -- replaced with /var/log/ceph/cluster2-client.mirror.0.83837.log.gz 2026-02-22T22:12:58.946 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.73732.log 2026-02-22T22:12:58.946 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.53089.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58465.log 2026-02-22T22:12:58.946 INFO:teuthology.orchestra.run.trial026.stderr: 88.2% -- replaced with /var/log/ceph/cluster1-client.mirror.53089.log.gz 2026-02-22T22:12:58.946 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.73732.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66506.log 2026-02-22T22:12:58.946 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.58465.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.58465.log.gz 2026-02-22T22:12:58.950 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68729.log 2026-02-22T22:12:58.950 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.66506.log: 92.5% -- replaced with /var/log/ceph/cluster1-client.mirror.73732.log.gz 2026-02-22T22:12:58.950 INFO:teuthology.orchestra.run.trial026.stderr:gzip 93.9% -- replaced with /var/log/ceph/cluster2-client.mirror.52723.log.gz 2026-02-22T22:12:58.950 INFO:teuthology.orchestra.run.trial026.stderr: -5 --verbose -- /var/log/ceph/cluster1-client.admin.52364.log 2026-02-22T22:12:58.950 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54140.log 2026-02-22T22:12:58.950 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.68729.log: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.66506.log.gz 2026-02-22T22:12:58.950 INFO:teuthology.orchestra.run.trial026.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.68729.log.gz/var/log/ceph/cluster1-client.admin.52364.log: 2026-02-22T22:12:58.950 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75355.log 87.8% -- replaced with /var/log/ceph/cluster1-client.admin.52364.log.gz 2026-02-22T22:12:58.950 INFO:teuthology.orchestra.run.trial026.stderr: 2026-02-22T22:12:58.950 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.54140.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77027.log 2026-02-22T22:12:58.950 INFO:teuthology.orchestra.run.trial026.stderr: 89.3% -- replaced with /var/log/ceph/cluster1-client.mirror.54140.log.gz 2026-02-22T22:12:58.950 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.75355.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57990.log 2026-02-22T22:12:58.950 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78004.log 2026-02-22T22:12:58.954 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.77027.log: /var/log/ceph/cluster2-client.mirror.57990.log: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.75355.log.gz 2026-02-22T22:12:58.954 INFO:teuthology.orchestra.run.trial026.stderr: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.57990.log.gz 2026-02-22T22:12:58.954 INFO:teuthology.orchestra.run.trial026.stderr:gzip 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.77027.log.gz 2026-02-22T22:12:58.954 INFO:teuthology.orchestra.run.trial026.stderr: -5 --verbose -- /var/log/ceph/cluster1-client.mirror.74616.log 2026-02-22T22:12:58.954 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.78004.log: 2.6% -- replaced with /var/log/ceph/cluster1-client.mirror.78004.log.gz 2026-02-22T22:12:58.954 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55804.log 2026-02-22T22:12:58.954 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.74616.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66485.log 2026-02-22T22:12:58.954 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.55804.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66389.log 2026-02-22T22:12:58.954 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.66485.log: 92.2% -- replaced with /var/log/ceph/cluster1-client.mirror.74616.log.gz 2026-02-22T22:12:58.954 INFO:teuthology.orchestra.run.trial026.stderr: 88.0% -- replaced with /var/log/ceph/cluster1-client.mirror.66485.log.gz 2026-02-22T22:12:58.954 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75107.log 2026-02-22T22:12:58.954 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.66389.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74479.log 2026-02-22T22:12:58.954 INFO:teuthology.orchestra.run.trial026.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.66389.log.gz 2026-02-22T22:12:58.955 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.75107.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72419.log 2026-02-22T22:12:58.958 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.74479.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.74479.log.gz 2026-02-22T22:12:58.958 INFO:teuthology.orchestra.run.trial026.stderr: 93.5%gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.72258.log 2026-02-22T22:12:58.958 INFO:teuthology.orchestra.run.trial026.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.55804.log.gz 2026-02-22T22:12:58.958 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.72419.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.63223.log 2026-02-22T22:12:58.958 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.72258.log: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.75107.log.gz 2026-02-22T22:12:58.958 INFO:teuthology.orchestra.run.trial026.stderr: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.72419.log.gz 2026-02-22T22:12:58.958 INFO:teuthology.orchestra.run.trial026.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.72258.log.gz 2026-02-22T22:12:58.958 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72234.log 2026-02-22T22:12:58.958 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.63223.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.1.74811.log 88.0% -- replaced with /var/log/ceph/cluster1-client.mirror.63223.log.gz 2026-02-22T22:12:58.958 INFO:teuthology.orchestra.run.trial026.stderr: 2026-02-22T22:12:58.958 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.72234.log: gzip -5 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.72234.log.gz 2026-02-22T22:12:58.958 INFO:teuthology.orchestra.run.trial026.stderr: --verbose -- /var/log/ceph/cluster2-client.mirror.61668.log 2026-02-22T22:12:58.958 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.1.74811.log: 93.9%gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78025.log 2026-02-22T22:12:58.958 INFO:teuthology.orchestra.run.trial026.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.54340.log.gz 2026-02-22T22:12:58.959 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.61668.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58487.log 2026-02-22T22:12:58.959 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5/var/log/ceph/cluster1-client.mirror.78025.log: --verbose -- /var/log/ceph/cluster1-client.mirror.80230.log 2026-02-22T22:12:58.959 INFO:teuthology.orchestra.run.trial026.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.78025.log.gz 2026-02-22T22:12:58.960 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.58487.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.80725.log 2026-02-22T22:12:58.960 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.80230.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.80230.log.gz 2026-02-22T22:12:58.960 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59209.log 2026-02-22T22:12:58.960 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.80725.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.80725.log.gz 2026-02-22T22:12:58.960 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59226.log 2026-02-22T22:12:58.960 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.59209.log: 79.5% -- replaced with /var/log/ceph/cluster2-client.mirror.59209.log.gz 2026-02-22T22:12:58.961 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74813.log 2026-02-22T22:12:58.961 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.59226.log: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.59226.log.gz 2026-02-22T22:12:58.962 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71947.log 2026-02-22T22:12:58.962 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.74813.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.74813.log.gz 2026-02-22T22:12:58.962 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68246.log 2026-02-22T22:12:58.963 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.71947.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.71947.log.gz 2026-02-22T22:12:58.964 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55503.log 2026-02-22T22:12:58.964 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.68246.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.68246.log.gz 2026-02-22T22:12:58.964 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77399.log 2026-02-22T22:12:58.967 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.55503.log: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.55503.log.gz 2026-02-22T22:12:58.968 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76420.log 2026-02-22T22:12:58.969 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.77399.log: 93.1% -- replaced with /var/log/ceph/cluster2-client.mirror.1.74811.log.gz 2026-02-22T22:12:58.969 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54318.log 2026-02-22T22:12:58.969 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.76420.log: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.77399.log.gz 2026-02-22T22:12:58.970 INFO:teuthology.orchestra.run.trial026.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.76420.log.gz 2026-02-22T22:12:58.970 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.71925.log 2026-02-22T22:12:58.970 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.54318.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.47449.log 2026-02-22T22:12:58.971 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.71925.log: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.54318.log.gz 2026-02-22T22:12:58.971 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77379.log 2026-02-22T22:12:58.971 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.admin.47449.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.47449.log.gz 2026-02-22T22:12:58.972 INFO:teuthology.orchestra.run.trial026.stderr: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.71925.log.gz 2026-02-22T22:12:58.972 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75397.log 2026-02-22T22:12:58.972 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.77379.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57604.log 2026-02-22T22:12:58.972 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.75397.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.77379.log.gz 2026-02-22T22:12:58.973 INFO:teuthology.orchestra.run.trial026.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.75397.log.gz 2026-02-22T22:12:58.973 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52616.log 2026-02-22T22:12:58.973 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.57604.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.48578.log 2026-02-22T22:12:58.974 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.52616.log: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.57780.log.gz 2026-02-22T22:12:58.974 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52024.log 2026-02-22T22:12:58.975 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.admin.48578.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.48578.log.gz 2026-02-22T22:12:58.975 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79780.log 2026-02-22T22:12:58.976 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.52024.log: 93.6% -- replaced with /var/log/ceph/cluster2-client.mirror.52616.log.gz 2026-02-22T22:12:58.979 INFO:teuthology.orchestra.run.trial026.stderr: 91.6% -- replaced with /var/log/ceph/cluster2-client.mirror.52024.log.gz 2026-02-22T22:12:58.979 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69219.log 2026-02-22T22:12:58.979 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.79780.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54253.log 2026-02-22T22:12:58.979 INFO:teuthology.orchestra.run.trial026.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.79780.log.gz 2026-02-22T22:12:58.979 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.69219.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.64922.log 2026-02-22T22:12:58.979 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.54253.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.69219.log.gz 2026-02-22T22:12:58.979 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.72113.log 2026-02-22T22:12:58.981 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.0.64922.log: 93.4% -- replaced with /var/log/ceph/cluster1-client.mirror.54253.log.gz 2026-02-22T22:12:58.990 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.54374.log 2026-02-22T22:12:58.991 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.72113.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.72113.log.gz 2026-02-22T22:12:58.991 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-mon.a.log 2026-02-22T22:12:58.992 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.admin.54374.log: 88.0% -- replaced with /var/log/ceph/cluster1-client.admin.54374.log.gz 2026-02-22T22:12:58.992 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54604.log 2026-02-22T22:12:59.001 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-mon.a.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52449.log 2026-02-22T22:12:59.002 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.54604.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.54604.log.gz 2026-02-22T22:12:59.007 INFO:teuthology.orchestra.run.trial026.stderr: 93.9% -- replaced with /var/log/ceph/cluster2-client.mirror.57151.log.gz 2026-02-22T22:12:59.008 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57194.log 2026-02-22T22:12:59.009 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.52449.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.52449.log.gz 2026-02-22T22:12:59.009 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.74391.log 2026-02-22T22:12:59.010 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.57194.log: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.57194.log.gz 2026-02-22T22:12:59.010 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.62785.log 2026-02-22T22:12:59.011 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.74391.log: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.74391.log.gz 2026-02-22T22:12:59.012 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55655.log 2026-02-22T22:12:59.012 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.62785.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.62785.log.gz 2026-02-22T22:12:59.013 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75727.log 2026-02-22T22:12:59.013 INFO:teuthology.orchestra.run.trial026.stderr: 93.9% -- replaced with /var/log/ceph/cluster2-client.mirror.57948.log.gz 2026-02-22T22:12:59.014 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.55655.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67701.log 2026-02-22T22:12:59.015 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.75727.log: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.75727.log.gz 2026-02-22T22:12:59.015 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48054.log 2026-02-22T22:12:59.016 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.67701.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.67701.log.gz 2026-02-22T22:12:59.017 INFO:teuthology.orchestra.run.trial026.stderr: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.55655.log.gz 2026-02-22T22:12:59.017 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.73643.log 2026-02-22T22:12:59.017 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77709.log 2026-02-22T22:12:59.017 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.73643.log: /var/log/ceph/cluster2-client.mirror.48054.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.73643.log.gz 2026-02-22T22:12:59.018 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.62556.log 2026-02-22T22:12:59.019 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.77709.log: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.77709.log.gz 2026-02-22T22:12:59.020 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.48054.log 2026-02-22T22:12:59.020 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.62556.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.62556.log.gz 2026-02-22T22:12:59.020 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69641.log 2026-02-22T22:12:59.027 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.1.48054.log: 93.0% -- replaced with /var/log/ceph/cluster1-client.mirror.0.64922.log.gz 2026-02-22T22:12:59.028 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63244.log 2026-02-22T22:12:59.028 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.69641.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.69641.log.gz 2026-02-22T22:12:59.029 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73086.log 2026-02-22T22:12:59.030 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.63244.log: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.63244.log.gz 2026-02-22T22:12:59.030 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52417.log 2026-02-22T22:12:59.031 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.73086.log: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.73086.log.gz 2026-02-22T22:12:59.032 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50752.log 2026-02-22T22:12:59.033 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.52417.log: 91.6% -- replaced with /var/log/ceph/cluster2-client.mirror.52417.log.gz 2026-02-22T22:12:59.033 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.73864.log 2026-02-22T22:12:59.045 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.50752.log: 94.6% -- replaced with /var/log/ceph/cluster1-client.mirror.1.48054.log.gz 2026-02-22T22:12:59.045 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55870.log 2026-02-22T22:12:59.046 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.73864.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.48602.log 2026-02-22T22:12:59.046 INFO:teuthology.orchestra.run.trial026.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.73864.log.gz 2026-02-22T22:12:59.048 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.55870.log: 94.6% -- replaced with /var/log/ceph/cluster2-client.mirror.48054.log.gz 2026-02-22T22:12:59.049 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63573.log 2026-02-22T22:12:59.049 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.admin.48602.log: 79.9% -- replaced with /var/log/ceph/cluster1-client.admin.48602.log.gz 2026-02-22T22:12:59.049 INFO:teuthology.orchestra.run.trial026.stderr: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.55870.log.gz 2026-02-22T22:12:59.051 INFO:teuthology.orchestra.run.trial026.stderr: 93.9% -- replaced with /var/log/ceph/cluster2-client.mirror.84345.log.gz 2026-02-22T22:12:59.054 INFO:teuthology.orchestra.run.trial026.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.58118.log.gz 2026-02-22T22:12:59.074 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67263.log 2026-02-22T22:12:59.074 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.63573.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.85472.log 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.63573.log.gz 2026-02-22T22:12:59.074 INFO:teuthology.orchestra.run.trial026.stderr: 2026-02-22T22:12:59.075 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.67263.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.46473.log 2026-02-22T22:12:59.075 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.admin.85472.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.85472.log.gz 2026-02-22T22:12:59.075 INFO:teuthology.orchestra.run.trial026.stderr:gzip 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.67263.log.gz 2026-02-22T22:12:59.075 INFO:teuthology.orchestra.run.trial026.stderr: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75934.log 2026-02-22T22:12:59.075 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.admin.46473.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.46473.log.gz 2026-02-22T22:12:59.075 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.71301.log 2026-02-22T22:12:59.075 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.75934.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84785.log 2026-02-22T22:12:59.075 INFO:teuthology.orchestra.run.trial026.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.75934.log.gz 2026-02-22T22:12:59.075 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.71301.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54578.log 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.71301.log.gz 2026-02-22T22:12:59.075 INFO:teuthology.orchestra.run.trial026.stderr: 2026-02-22T22:12:59.075 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.84785.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.mirror.84785.log.gz 2026-02-22T22:12:59.075 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.0.51571.log 2026-02-22T22:12:59.075 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.54578.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59716.log 2026-02-22T22:12:59.075 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.0.51571.log: 90.0%gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78318.log 2026-02-22T22:12:59.075 INFO:teuthology.orchestra.run.trial026.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.54578.log.gz 2026-02-22T22:12:59.075 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.59716.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77647.log 2026-02-22T22:12:59.075 INFO:teuthology.orchestra.run.trial026.stderr: 93.8%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82924.log 2026-02-22T22:12:59.075 INFO:teuthology.orchestra.run.trial026.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.58742.log.gz 2026-02-22T22:12:59.075 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.78318.log: /var/log/ceph/cluster2-client.mirror.77647.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.78318.log.gz 2026-02-22T22:12:59.075 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78655.log 2026-02-22T22:12:59.075 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83224.log 2026-02-22T22:12:59.076 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.82924.log: /var/log/ceph/cluster1-client.mirror.78655.log: 79.7% -- replaced with /var/log/ceph/cluster2-client.mirror.82924.log.gz 2026-02-22T22:12:59.076 INFO:teuthology.orchestra.run.trial026.stderr: 93.2% -- replaced with /var/log/ceph/cluster2-client.mirror.0.51571.log.gz 2026-02-22T22:12:59.076 INFO:teuthology.orchestra.run.trial026.stderr: 86.6% -- replaced with /var/log/ceph/cluster1-client.mirror.78655.log.gz 2026-02-22T22:12:59.076 INFO:teuthology.orchestra.run.trial026.stderr: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.77647.log.gz 2026-02-22T22:12:59.076 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79330.log 2026-02-22T22:12:59.076 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.83224.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84245.log 2026-02-22T22:12:59.076 INFO:teuthology.orchestra.run.trial026.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.83224.log.gz 2026-02-22T22:12:59.076 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.79330.log: gzip 86.6% -- replaced with /var/log/ceph/cluster1-client.mirror.79330.log.gz 2026-02-22T22:12:59.076 INFO:teuthology.orchestra.run.trial026.stderr: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69288.log 2026-02-22T22:12:59.076 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.84245.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.46763.log 2026-02-22T22:12:59.076 INFO:teuthology.orchestra.run.trial026.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.84245.log.gz 2026-02-22T22:12:59.076 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.69288.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.68519.log 2026-02-22T22:12:59.076 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.admin.46763.log: gzip -5 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.46763.log.gz 2026-02-22T22:12:59.076 INFO:teuthology.orchestra.run.trial026.stderr: --verbose -- /var/log/ceph/cluster2-client.mirror.57035.log 2026-02-22T22:12:59.076 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.1.68519.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.47645.log 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.69288.log.gz 2026-02-22T22:12:59.076 INFO:teuthology.orchestra.run.trial026.stderr: 2026-02-22T22:12:59.076 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.57035.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62920.log 2026-02-22T22:12:59.076 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.admin.47645.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.47645.log.gz 2026-02-22T22:12:59.076 INFO:teuthology.orchestra.run.trial026.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.57035.log.gz 2026-02-22T22:12:59.076 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67647.log 2026-02-22T22:12:59.076 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.62920.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68174.log 2026-02-22T22:12:59.076 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.67647.log: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.62920.log.gz 2026-02-22T22:12:59.077 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57675.log 2026-02-22T22:12:59.077 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.68174.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.68174.log.gz 2026-02-22T22:12:59.077 INFO:teuthology.orchestra.run.trial026.stderr: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.67647.log.gz 2026-02-22T22:12:59.077 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64469.log 2026-02-22T22:12:59.077 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.57675.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84303.log 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.57675.log.gz 2026-02-22T22:12:59.077 INFO:teuthology.orchestra.run.trial026.stderr: 2026-02-22T22:12:59.077 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.64469.log: gzip -5 --verbose 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.64469.log.gz 2026-02-22T22:12:59.077 INFO:teuthology.orchestra.run.trial026.stderr: -- /var/log/ceph/cluster2-client.mirror.72017.log 2026-02-22T22:12:59.077 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.84303.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75955.log 2026-02-22T22:12:59.077 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.72017.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.84303.log.gz 2026-02-22T22:12:59.077 INFO:teuthology.orchestra.run.trial026.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.72017.log.gz 2026-02-22T22:12:59.077 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67156.log 2026-02-22T22:12:59.077 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.75955.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82941.log 2026-02-22T22:12:59.077 INFO:teuthology.orchestra.run.trial026.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.75955.log.gz 2026-02-22T22:12:59.077 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.67156.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68311.log 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.67156.log.gz 2026-02-22T22:12:59.077 INFO:teuthology.orchestra.run.trial026.stderr: 2026-02-22T22:12:59.077 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.82941.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84159.log 2026-02-22T22:12:59.077 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.68311.log: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.82941.log.gz 2026-02-22T22:12:59.077 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83146.log 2026-02-22T22:12:59.077 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.84159.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.84159.log.gz 2026-02-22T22:12:59.077 INFO:teuthology.orchestra.run.trial026.stderr: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.68311.log.gz 2026-02-22T22:12:59.077 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54092.log 2026-02-22T22:12:59.077 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.83146.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70940.log 2026-02-22T22:12:59.078 INFO:teuthology.orchestra.run.trial026.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.83146.log.gz 2026-02-22T22:12:59.078 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.54092.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74704.log 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.54092.log.gz 2026-02-22T22:12:59.078 INFO:teuthology.orchestra.run.trial026.stderr: 2026-02-22T22:12:59.078 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.70940.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77771.log 2026-02-22T22:12:59.078 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.74704.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.70940.log.gz 2026-02-22T22:12:59.078 INFO:teuthology.orchestra.run.trial026.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.74704.log.gz 2026-02-22T22:12:59.078 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.48755.log 2026-02-22T22:12:59.078 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.77771.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.73370.log 2026-02-22T22:12:59.078 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.48755.log: 58.0% -- replaced with /var/log/ceph/cluster1-client.mirror.48755.log.gz 2026-02-22T22:12:59.078 INFO:teuthology.orchestra.run.trial026.stderr: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.77771.log.gz 2026-02-22T22:12:59.079 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72208.log 2026-02-22T22:12:59.079 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.1.73370.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70347.log 2026-02-22T22:12:59.079 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.72208.log: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.72208.log.gz 2026-02-22T22:12:59.080 INFO:teuthology.orchestra.run.trial026.stderr: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.1.68519.log.gz 2026-02-22T22:12:59.081 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.48293.log 2026-02-22T22:12:59.082 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.70347.log: gzip -5 --verbose 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.70347.log.gz 2026-02-22T22:12:59.082 INFO:teuthology.orchestra.run.trial026.stderr: -- /var/log/ceph/cluster1-client.mirror.51879.log 2026-02-22T22:12:59.082 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.48293.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.84806.log 2026-02-22T22:12:59.082 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.51879.log: 94.2% -- replaced with /var/log/ceph/cluster1-client.mirror.48293.log.gzgzip 2026-02-22T22:12:59.082 INFO:teuthology.orchestra.run.trial026.stderr: -5 --verbose -- /var/log/ceph/cluster1-client.mirror.74350.log 2026-02-22T22:12:59.083 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.admin.84806.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57236.log 2026-02-22T22:12:59.083 INFO:teuthology.orchestra.run.trial026.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.84806.log.gz 2026-02-22T22:12:59.083 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.74350.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52659.log 2026-02-22T22:12:59.083 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.57236.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.74350.log.gz 2026-02-22T22:12:59.084 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55976.log 2026-02-22T22:12:59.085 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.52659.log: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.51879.log.gz 2026-02-22T22:12:59.085 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67060.log 2026-02-22T22:12:59.087 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.55976.log: 93.9% 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.52659.log.gz 2026-02-22T22:12:59.087 INFO:teuthology.orchestra.run.trial026.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.59182.log.gz 2026-02-22T22:12:59.088 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58804.log 2026-02-22T22:12:59.088 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79893.log 2026-02-22T22:12:59.088 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.67060.log: /var/log/ceph/cluster2-client.mirror.58804.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.67060.log.gz 2026-02-22T22:12:59.089 INFO:teuthology.orchestra.run.trial026.stderr: 93.5% 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.55976.log.gz 2026-02-22T22:12:59.089 INFO:teuthology.orchestra.run.trial026.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.58804.log.gz 2026-02-22T22:12:59.089 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59929.log 2026-02-22T22:12:59.089 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71229.log 2026-02-22T22:12:59.090 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.79893.log: /var/log/ceph/cluster2-client.mirror.59929.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.79893.log.gzgzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77296.log 2026-02-22T22:12:59.090 INFO:teuthology.orchestra.run.trial026.stderr: 2026-02-22T22:12:59.091 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose 89.6% -- /var/log/ceph/cluster1-client.mirror.64814.log 2026-02-22T22:12:59.091 INFO:teuthology.orchestra.run.trial026.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.59929.log.gz 2026-02-22T22:12:59.092 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.71229.log: /var/log/ceph/cluster2-client.mirror.77296.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55999.log 2026-02-22T22:12:59.092 INFO:teuthology.orchestra.run.trial026.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.71229.log.gz 2026-02-22T22:12:59.092 INFO:teuthology.orchestra.run.trial026.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.77296.log.gz 2026-02-22T22:12:59.092 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.64814.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78745.log 2026-02-22T22:12:59.092 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.55999.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73754.log 2026-02-22T22:12:59.092 INFO:teuthology.orchestra.run.trial026.stderr: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.64814.log.gz 2026-02-22T22:12:59.092 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.78745.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.78745.log.gz 2026-02-22T22:12:59.094 INFO:teuthology.orchestra.run.trial026.stderr: 92.6% -- replaced with /var/log/ceph/cluster1-client.mirror.1.73370.log.gz 2026-02-22T22:12:59.096 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55546.log 2026-02-22T22:12:59.096 INFO:teuthology.orchestra.run.trial026.stderr: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.55999.log.gz 2026-02-22T22:12:59.096 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.73754.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68270.log 2026-02-22T22:12:59.096 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.55546.log: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.73754.log.gz 2026-02-22T22:12:59.096 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60029.log 2026-02-22T22:12:59.096 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.68270.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.80253.log 2026-02-22T22:12:59.096 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.60029.log: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.68270.log.gz 2026-02-22T22:12:59.096 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81976.log 2026-02-22T22:12:59.097 INFO:teuthology.orchestra.run.trial026.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.60029.log.gz 2026-02-22T22:12:59.097 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66461.log 2026-02-22T22:12:59.102 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.80253.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.80253.log.gz 2026-02-22T22:12:59.102 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.83951.log 2026-02-22T22:12:59.102 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.81976.log: /var/log/ceph/cluster1-client.mirror.66461.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.46914.log 2026-02-22T22:12:59.102 INFO:teuthology.orchestra.run.trial026.stderr: 86.9% 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.66461.log.gz 2026-02-22T22:12:59.102 INFO:teuthology.orchestra.run.trial026.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.81976.log.gz/var/log/ceph/cluster1-client.mirror.83951.log: 2026-02-22T22:12:59.102 INFO:teuthology.orchestra.run.trial026.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.83951.log.gz 2026-02-22T22:12:59.102 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.72975.log 2026-02-22T22:12:59.102 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.admin.46914.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.46914.log.gz 2026-02-22T22:12:59.102 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.48653.log 2026-02-22T22:12:59.102 INFO:teuthology.orchestra.run.trial026.stderr: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.55546.log.gz 2026-02-22T22:12:59.102 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.72975.log: gzip -5 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.72975.log.gz 2026-02-22T22:12:59.102 INFO:teuthology.orchestra.run.trial026.stderr: --verbose -- /var/log/ceph/cluster2-client.mirror.70056.log 2026-02-22T22:12:59.102 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.48653.log: 54.0% -- replaced with /var/log/ceph/cluster1-client.mirror.48653.log.gz 2026-02-22T22:12:59.102 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52547.log 2026-02-22T22:12:59.102 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.70056.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67534.log 2026-02-22T22:12:59.103 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.52547.log: gzip 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.70056.log.gz 2026-02-22T22:12:59.103 INFO:teuthology.orchestra.run.trial026.stderr: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76100.log 2026-02-22T22:12:59.103 INFO:teuthology.orchestra.run.trial026.stderr: 88.2% -- replaced with /var/log/ceph/cluster1-client.mirror.52547.log.gz 2026-02-22T22:12:59.103 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.67534.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76379.log 2026-02-22T22:12:59.103 INFO:teuthology.orchestra.run.trial026.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.67534.log.gz 2026-02-22T22:12:59.103 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.76100.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65863.log 2026-02-22T22:12:59.103 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.76379.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.72577.log 2026-02-22T22:12:59.103 INFO:teuthology.orchestra.run.trial026.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.76379.log.gz 2026-02-22T22:12:59.103 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.65863.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70395.log 2026-02-22T22:12:59.103 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68753.log 2026-02-22T22:12:59.103 INFO:teuthology.orchestra.run.trial026.stderr: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.76100.log.gz 2026-02-22T22:12:59.106 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.72577.log: /var/log/ceph/cluster1-client.mirror.70395.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.74575.log 2026-02-22T22:12:59.106 INFO:teuthology.orchestra.run.trial026.stderr: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.72577.log.gz 2026-02-22T22:12:59.106 INFO:teuthology.orchestra.run.trial026.stderr: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.65863.log.gz 88.0% -- replaced with /var/log/ceph/cluster1-client.mirror.70395.log.gz 2026-02-22T22:12:59.106 INFO:teuthology.orchestra.run.trial026.stderr: 2026-02-22T22:12:59.106 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.68753.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.74326.log 2026-02-22T22:12:59.106 INFO:teuthology.orchestra.run.trial026.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.68753.log.gz 2026-02-22T22:12:59.106 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.74575.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54276.log 2026-02-22T22:12:59.106 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.74326.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57721.log 2026-02-22T22:12:59.106 INFO:teuthology.orchestra.run.trial026.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.74575.log.gz 2026-02-22T22:12:59.106 INFO:teuthology.orchestra.run.trial026.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.74326.log.gz/var/log/ceph/cluster2-client.mirror.54276.log: 2026-02-22T22:12:59.106 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83420.log 2026-02-22T22:12:59.106 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.57721.log: 79.4% -- replaced with /var/log/ceph/cluster2-client.mirror.57721.log.gz 2026-02-22T22:12:59.106 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64652.log 2026-02-22T22:12:59.106 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.83420.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.72356.log 2026-02-22T22:12:59.106 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52223.log 2026-02-22T22:12:59.106 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.64652.log: /var/log/ceph/cluster1-client.mirror.72356.log: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.72356.log.gz 2026-02-22T22:12:59.107 INFO:teuthology.orchestra.run.trial026.stderr: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.64652.log.gz 2026-02-22T22:12:59.107 INFO:teuthology.orchestra.run.trial026.stderr:gzip 91.6% -- replaced with /var/log/ceph/cluster2-client.mirror.83420.log.gz 2026-02-22T22:12:59.107 INFO:teuthology.orchestra.run.trial026.stderr: -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67751.log 2026-02-22T22:12:59.108 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.52223.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68896.log 2026-02-22T22:12:59.108 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.67751.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.62175.log 2026-02-22T22:12:59.108 INFO:teuthology.orchestra.run.trial026.stderr: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.54276.log.gz 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.67751.log.gz 2026-02-22T22:12:59.108 INFO:teuthology.orchestra.run.trial026.stderr: 2026-02-22T22:12:59.108 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.68896.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.68896.log.gz 2026-02-22T22:12:59.109 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73144.log 2026-02-22T22:12:59.109 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.62175.log: 1.3% -- replaced with /var/log/ceph/cluster1-client.mirror.62175.log.gz 2026-02-22T22:12:59.112 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67957.log 2026-02-22T22:12:59.112 INFO:teuthology.orchestra.run.trial026.stderr: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.52223.log.gz 2026-02-22T22:12:59.112 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.73144.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.73192.log 2026-02-22T22:12:59.112 INFO:teuthology.orchestra.run.trial026.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.61830.log.gz 2026-02-22T22:12:59.112 INFO:teuthology.orchestra.run.trial026.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.73144.log.gz 2026-02-22T22:12:59.112 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.67957.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75190.log 2026-02-22T22:12:59.112 INFO:teuthology.orchestra.run.trial026.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.67957.log.gz 2026-02-22T22:12:59.112 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.73192.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59417.log 2026-02-22T22:12:59.112 INFO:teuthology.orchestra.run.trial026.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.73192.log.gz/var/log/ceph/cluster2-client.mirror.75190.log: 2026-02-22T22:12:59.112 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54188.log 2026-02-22T22:12:59.112 INFO:teuthology.orchestra.run.trial026.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.75190.log.gz 2026-02-22T22:12:59.112 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.59417.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62048.log 2026-02-22T22:12:59.112 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.54188.log: gzip -5 --verbose 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.59417.log.gz 2026-02-22T22:12:59.112 INFO:teuthology.orchestra.run.trial026.stderr: -- /var/log/ceph/cluster1-client.mirror.83745.log 2026-02-22T22:12:59.112 INFO:teuthology.orchestra.run.trial026.stderr: 89.3% -- replaced with /var/log/ceph/cluster1-client.mirror.54188.log.gz 2026-02-22T22:12:59.112 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.62048.log: gzip -5 79.6% -- replaced with /var/log/ceph/cluster2-client.mirror.62048.log.gz 2026-02-22T22:12:59.112 INFO:teuthology.orchestra.run.trial026.stderr: --verbose -- /var/log/ceph/cluster1-client.mirror.64630.log 2026-02-22T22:12:59.114 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.83745.log: gzip -5 --verbose 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.83745.log.gz 2026-02-22T22:12:59.114 INFO:teuthology.orchestra.run.trial026.stderr: -- /var/log/ceph/cluster2-client.mirror.66289.log 2026-02-22T22:12:59.115 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.64630.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.72041.log 2026-02-22T22:12:59.115 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.66289.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75045.log 2026-02-22T22:12:59.115 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.72041.log: gzip -5 87.8% --verbose -- /var/log/ceph/cluster2-client.mirror.69816.log 2026-02-22T22:12:59.115 INFO:teuthology.orchestra.run.trial026.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.72041.log.gz 2026-02-22T22:12:59.115 INFO:teuthology.orchestra.run.trial026.stderr: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.64630.log.gz 2026-02-22T22:12:59.115 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.75045.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71726.log 2026-02-22T22:12:59.115 INFO:teuthology.orchestra.run.trial026.stderr: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.66289.log.gz 2026-02-22T22:12:59.115 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.69816.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65080.log 2026-02-22T22:12:59.115 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.71726.log: 89.8%gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79578.log 2026-02-22T22:12:59.115 INFO:teuthology.orchestra.run.trial026.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.69816.log.gz 2026-02-22T22:12:59.117 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.65080.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61398.log 2026-02-22T22:12:59.117 INFO:teuthology.orchestra.run.trial026.stderr: 89.8% 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.75045.log.gz 2026-02-22T22:12:59.117 INFO:teuthology.orchestra.run.trial026.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.71726.log.gz 2026-02-22T22:12:59.117 INFO:teuthology.orchestra.run.trial026.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.65080.log.gz 2026-02-22T22:12:59.118 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57865.log 2026-02-22T22:12:59.118 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.79578.log: /var/log/ceph/cluster2-client.mirror.61398.log: 87.0%gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52178.log 2026-02-22T22:12:59.118 INFO:teuthology.orchestra.run.trial026.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.79578.log.gz 2026-02-22T22:12:59.118 INFO:teuthology.orchestra.run.trial026.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.61398.log.gz 2026-02-22T22:12:59.118 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.57865.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.72999.log 2026-02-22T22:12:59.118 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.52178.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68291.log 2026-02-22T22:12:59.118 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.72999.log: 88.5% -- replaced with /var/log/ceph/cluster1-client.mirror.52178.log.gz 2026-02-22T22:12:59.118 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.72999.log.gz 2026-02-22T22:12:59.118 INFO:teuthology.orchestra.run.trial026.stderr: /var/log/ceph/cluster1-client.mirror.51224.log 2026-02-22T22:12:59.118 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.68291.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69665.log 2026-02-22T22:12:59.118 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.51224.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75459.log 2026-02-22T22:12:59.118 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.69665.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66437.log 2026-02-22T22:12:59.119 INFO:teuthology.orchestra.run.trial026.stderr: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.68291.log.gz 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.69665.log.gz 2026-02-22T22:12:59.119 INFO:teuthology.orchestra.run.trial026.stderr: 2026-02-22T22:12:59.119 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.75459.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73816.log 2026-02-22T22:12:59.119 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.66437.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.75459.log.gz 2026-02-22T22:12:59.119 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65995.log 2026-02-22T22:12:59.119 INFO:teuthology.orchestra.run.trial026.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.66437.log.gz 2026-02-22T22:12:59.119 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56487.log 2026-02-22T22:12:59.119 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.73816.log: /var/log/ceph/cluster1-client.mirror.65995.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58596.log 2026-02-22T22:12:59.120 INFO:teuthology.orchestra.run.trial026.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.73816.log.gz 2026-02-22T22:12:59.120 INFO:teuthology.orchestra.run.trial026.stderr: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.65995.log.gz 2026-02-22T22:12:59.120 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.56487.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81032.log 2026-02-22T22:12:59.120 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.58596.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.56487.log.gz 2026-02-22T22:12:59.121 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.74551.log 2026-02-22T22:12:59.121 INFO:teuthology.orchestra.run.trial026.stderr: 79.5% -- replaced with /var/log/ceph/cluster2-client.mirror.58596.log.gz 2026-02-22T22:12:59.121 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.81032.log: 93.3% -- replaced with /var/log/ceph/cluster1-client.mirror.51224.log.gz 2026-02-22T22:12:59.121 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83166.log 2026-02-22T22:12:59.121 INFO:teuthology.orchestra.run.trial026.stderr: 80.7% -- replaced with /var/log/ceph/cluster1-client.mirror.81032.log.gz 2026-02-22T22:12:59.121 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.73619.log 2026-02-22T22:12:59.121 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.74551.log: /var/log/ceph/cluster2-client.mirror.83166.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82961.log 2026-02-22T22:12:59.121 INFO:teuthology.orchestra.run.trial026.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.74551.log.gz 2026-02-22T22:12:59.121 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.2.71472.log 2026-02-22T22:12:59.122 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.73619.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82790.log 2026-02-22T22:12:59.122 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.82961.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.73619.log.gz 2026-02-22T22:12:59.122 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.2.71472.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.82961.log.gz 2026-02-22T22:12:59.122 INFO:teuthology.orchestra.run.trial026.stderr: 91.7% -- replaced with /var/log/ceph/cluster2-client.mirror.83166.log.gz 2026-02-22T22:12:59.123 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61563.log 2026-02-22T22:12:59.123 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.82790.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70576.log 2026-02-22T22:12:59.123 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.61563.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61585.log 2026-02-22T22:12:59.124 INFO:teuthology.orchestra.run.trial026.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.61563.log.gz/var/log/ceph/cluster1-client.mirror.70576.log: 2026-02-22T22:12:59.124 INFO:teuthology.orchestra.run.trial026.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.70576.log.gz 2026-02-22T22:12:59.125 INFO:teuthology.orchestra.run.trial026.stderr: 91.6% -- replaced with /var/log/ceph/cluster2-client.mirror.82790.log.gz 2026-02-22T22:12:59.126 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53453.log 2026-02-22T22:12:59.127 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.61585.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75645.log 2026-02-22T22:12:59.127 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.53453.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78093.log 2026-02-22T22:12:59.127 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.75645.log: 89.9% 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.75645.log.gz 2026-02-22T22:12:59.128 INFO:teuthology.orchestra.run.trial026.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.53453.log.gz 2026-02-22T22:12:59.128 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.71060.log 2026-02-22T22:12:59.129 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.78093.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75066.log 2026-02-22T22:12:59.129 INFO:teuthology.orchestra.run.trial026.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.78093.log.gz 2026-02-22T22:12:59.129 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.71060.log: 87.8%gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.83768.log 2026-02-22T22:12:59.129 INFO:teuthology.orchestra.run.trial026.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.71060.log.gz 2026-02-22T22:12:59.129 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.75066.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65614.log 2026-02-22T22:12:59.130 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.83768.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.75066.log.gz 2026-02-22T22:12:59.130 INFO:teuthology.orchestra.run.trial026.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.83768.log.gz 2026-02-22T22:12:59.130 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.73840.log 2026-02-22T22:12:59.130 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.65614.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67406.log 2026-02-22T22:12:59.131 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.73840.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.73840.log.gz 2026-02-22T22:12:59.131 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60051.log 2026-02-22T22:12:59.131 INFO:teuthology.orchestra.run.trial026.stderr: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.65614.log.gz 2026-02-22T22:12:59.132 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.67406.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60009.log 2026-02-22T22:12:59.132 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.60051.log: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.67406.log.gz 2026-02-22T22:12:59.134 INFO:teuthology.orchestra.run.trial026.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.84093.log.gz 2026-02-22T22:12:59.134 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.48832.log 2026-02-22T22:12:59.135 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.60009.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67886.log 2026-02-22T22:12:59.135 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.admin.48832.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.48832.log.gz 2026-02-22T22:12:59.135 INFO:teuthology.orchestra.run.trial026.stderr: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.60009.log.gz 2026-02-22T22:12:59.135 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70082.log 2026-02-22T22:12:59.136 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.67886.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56407.log 2026-02-22T22:12:59.136 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.70082.log: 87.7% 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.67886.log.gz 2026-02-22T22:12:59.136 INFO:teuthology.orchestra.run.trial026.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.70082.log.gz 2026-02-22T22:12:59.136 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51267.log 2026-02-22T22:12:59.137 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.56407.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66809.log 2026-02-22T22:12:59.137 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.51267.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.56407.log.gz 2026-02-22T22:12:59.138 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61193.log 2026-02-22T22:12:59.138 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.66809.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.66809.log.gz 2026-02-22T22:12:59.139 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81625.log 2026-02-22T22:12:59.139 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.61193.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.61193.log.gz 2026-02-22T22:12:59.140 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66907.log 2026-02-22T22:12:59.140 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.81625.log: 93.5% 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.81625.log.gz 2026-02-22T22:12:59.140 INFO:teuthology.orchestra.run.trial026.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.51267.log.gz 2026-02-22T22:12:59.140 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.63151.log 2026-02-22T22:12:59.141 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.66907.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.47743.log 2026-02-22T22:12:59.141 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.63151.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.66907.log.gz 2026-02-22T22:12:59.141 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65471.log 2026-02-22T22:12:59.141 INFO:teuthology.orchestra.run.trial026.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.63151.log.gz 2026-02-22T22:12:59.142 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.admin.47743.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.47743.log.gz 2026-02-22T22:12:59.142 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77151.log 2026-02-22T22:12:59.142 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.65471.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53527.log 2026-02-22T22:12:59.142 INFO:teuthology.orchestra.run.trial026.stderr: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.2.71472.log.gz 2026-02-22T22:12:59.143 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.77151.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84760.log 2026-02-22T22:12:59.143 INFO:teuthology.orchestra.run.trial026.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.65471.log.gz 2026-02-22T22:12:59.143 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.53527.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.73953.log 2026-02-22T22:12:59.143 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.84760.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.mirror.84760.log.gz 2026-02-22T22:12:59.144 INFO:teuthology.orchestra.run.trial026.stderr: 89.1% -- replaced with /var/log/ceph/cluster1-client.mirror.53527.log.gz 2026-02-22T22:12:59.144 INFO:teuthology.orchestra.run.trial026.stderr:gzip 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.77151.log.gz 2026-02-22T22:12:59.144 INFO:teuthology.orchestra.run.trial026.stderr: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74413.log 2026-02-22T22:12:59.144 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.73953.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76573.log 2026-02-22T22:12:59.144 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.74413.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54473.log 2026-02-22T22:12:59.145 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.76573.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.74413.log.gz 2026-02-22T22:12:59.145 INFO:teuthology.orchestra.run.trial026.stderr: 92.2% -- replaced with /var/log/ceph/cluster1-client.mirror.73953.log.gz 2026-02-22T22:12:59.145 INFO:teuthology.orchestra.run.trial026.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.76573.log.gz 2026-02-22T22:12:59.145 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56910.log 2026-02-22T22:12:59.146 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.54473.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76738.log 2026-02-22T22:12:59.146 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.56910.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.63448.log 2026-02-22T22:12:59.146 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.76738.log: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.56910.log.gz 2026-02-22T22:12:59.146 INFO:teuthology.orchestra.run.trial026.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.76738.log.gz 2026-02-22T22:12:59.147 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71129.log 2026-02-22T22:12:59.147 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59629.log 2026-02-22T22:12:59.147 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.63448.log: /var/log/ceph/cluster2-client.mirror.71129.log: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.63448.log.gz 2026-02-22T22:12:59.148 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75231.log 2026-02-22T22:12:59.148 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.59629.log: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.71129.log.gz 2026-02-22T22:12:59.153 INFO:teuthology.orchestra.run.trial026.stderr: 94.0% -- replaced with /var/log/ceph/cluster2-client.mirror.54473.log.gz 2026-02-22T22:12:59.153 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77337.log 2026-02-22T22:12:59.154 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.75231.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.63669.log 2026-02-22T22:12:59.155 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.77337.log: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.75231.log.gz 2026-02-22T22:12:59.155 INFO:teuthology.orchestra.run.trial026.stderr: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.77337.log.gz 2026-02-22T22:12:59.156 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.47227.log 2026-02-22T22:12:59.156 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.63669.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55890.log 2026-02-22T22:12:59.156 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.admin.47227.log: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.63669.log.gz 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.47227.log.gz 2026-02-22T22:12:59.156 INFO:teuthology.orchestra.run.trial026.stderr: 2026-02-22T22:12:59.157 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.48670.log 2026-02-22T22:12:59.157 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.55890.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.85325.log 2026-02-22T22:12:59.157 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.48670.log: 3.5% -- replaced with /var/log/ceph/cluster1-client.mirror.48670.log.gz 2026-02-22T22:12:59.160 INFO:teuthology.orchestra.run.trial026.stderr: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.55890.log.gz 2026-02-22T22:12:59.160 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52286.log 2026-02-22T22:12:59.160 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.47351.log 2026-02-22T22:12:59.161 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.admin.85325.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.85325.log.gz 2026-02-22T22:12:59.161 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.52286.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51746.log 2026-02-22T22:12:59.161 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.admin.47351.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.47351.log.gz 2026-02-22T22:12:59.161 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76552.log 2026-02-22T22:12:59.162 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.51746.log: 88.2% -- replaced with /var/log/ceph/cluster1-client.mirror.51746.log.gz 2026-02-22T22:12:59.162 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.71591.log 2026-02-22T22:12:59.163 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.76552.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.76552.log.gz 2026-02-22T22:12:59.163 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57323.log 2026-02-22T22:12:59.164 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.71591.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.71591.log.gz 2026-02-22T22:12:59.164 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55933.log 2026-02-22T22:12:59.164 INFO:teuthology.orchestra.run.trial026.stderr: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.52286.log.gz 2026-02-22T22:12:59.174 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.57323.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78610.log 2026-02-22T22:12:59.178 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.55933.log: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.55933.log.gz 2026-02-22T22:12:59.181 INFO:teuthology.orchestra.run.trial026.stderr: 93.9% -- replaced with /var/log/ceph/cluster2-client.mirror.58322.log.gz 2026-02-22T22:12:59.187 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48910.log 2026-02-22T22:12:59.188 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.78610.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71012.log 2026-02-22T22:12:59.188 INFO:teuthology.orchestra.run.trial026.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.78610.log.gz 2026-02-22T22:12:59.189 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.48910.log: 53.5%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56427.log 2026-02-22T22:12:59.189 INFO:teuthology.orchestra.run.trial026.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.48910.log.gz 2026-02-22T22:12:59.189 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.71012.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57823.log 2026-02-22T22:12:59.189 INFO:teuthology.orchestra.run.trial026.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.71012.log.gz 2026-02-22T22:12:59.189 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.56427.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76490.log 2026-02-22T22:12:59.189 INFO:teuthology.orchestra.run.trial026.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.56427.log.gz 2026-02-22T22:12:59.190 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.57823.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74011.log 2026-02-22T22:12:59.190 INFO:teuthology.orchestra.run.trial026.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.57823.log.gz 2026-02-22T22:12:59.191 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.76490.log: gzip -5 87.9% --verbose -- /var/log/ceph/cluster2-client.mirror.74453.log 2026-02-22T22:12:59.191 INFO:teuthology.orchestra.run.trial026.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.76490.log.gz 2026-02-22T22:12:59.191 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.74011.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63772.log 2026-02-22T22:12:59.191 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.74453.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.74011.log.gz 2026-02-22T22:12:59.192 INFO:teuthology.orchestra.run.trial026.stderr: 89.8%gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79848.log 2026-02-22T22:12:59.192 INFO:teuthology.orchestra.run.trial026.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.74453.log.gz 2026-02-22T22:12:59.192 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.63772.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61443.log 2026-02-22T22:12:59.193 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.79848.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.79848.log.gz 2026-02-22T22:12:59.193 INFO:teuthology.orchestra.run.trial026.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.63772.log.gz 2026-02-22T22:12:59.193 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69713.log 2026-02-22T22:12:59.193 INFO:teuthology.orchestra.run.trial026.stderr: 93.9% -- replaced with /var/log/ceph/cluster2-client.mirror.51311.log.gz 2026-02-22T22:12:59.194 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.61443.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63469.log 2026-02-22T22:12:59.194 INFO:teuthology.orchestra.run.trial026.stderr: 79.5% -- replaced with /var/log/ceph/cluster2-client.mirror.61443.log.gz 2026-02-22T22:12:59.194 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.69713.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.48270.log 2026-02-22T22:12:59.194 INFO:teuthology.orchestra.run.trial026.stderr: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.69713.log.gzgzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.72089.log 2026-02-22T22:12:59.194 INFO:teuthology.orchestra.run.trial026.stderr: 2026-02-22T22:12:59.194 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.63469.log: /var/log/ceph/cluster1-client.mirror.48270.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78520.log 2026-02-22T22:12:59.195 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.72089.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.72089.log.gz 2026-02-22T22:12:59.196 INFO:teuthology.orchestra.run.trial026.stderr: 94.0% -- replaced with /var/log/ceph/cluster1-client.mirror.48270.log.gz 2026-02-22T22:12:59.196 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84735.log 2026-02-22T22:12:59.196 INFO:teuthology.orchestra.run.trial026.stderr: 92.2%/var/log/ceph/cluster1-client.mirror.78520.log: -- replaced with /var/log/ceph/cluster2-client.mirror.63469.log.gz 2026-02-22T22:12:59.196 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75438.log 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.78520.log.gz 2026-02-22T22:12:59.196 INFO:teuthology.orchestra.run.trial026.stderr: 2026-02-22T22:12:59.196 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.84735.log: 0.0%gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.48806.log 2026-02-22T22:12:59.196 INFO:teuthology.orchestra.run.trial026.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.84735.log.gz 2026-02-22T22:12:59.197 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.75438.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72706.log 2026-02-22T22:12:59.199 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.48687.log 2026-02-22T22:12:59.200 INFO:teuthology.orchestra.run.trial026.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.75438.log.gz 2026-02-22T22:12:59.200 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.admin.48806.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.48806.log.gz 2026-02-22T22:12:59.200 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.72706.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59912.log 2026-02-22T22:12:59.200 INFO:teuthology.orchestra.run.trial026.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.72706.log.gz 2026-02-22T22:12:59.200 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.48687.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.0.50819.log 2026-02-22T22:12:59.200 INFO:teuthology.orchestra.run.trial026.stderr: 3.5% -- replaced with /var/log/ceph/cluster1-client.mirror.48687.log.gz 2026-02-22T22:12:59.200 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70898.log 2026-02-22T22:12:59.200 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.59912.log: 79.7%/var/log/ceph/cluster2-client.mirror.0.50819.log: -- replaced with /var/log/ceph/cluster2-client.mirror.59912.log.gz 2026-02-22T22:12:59.200 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51698.log 2026-02-22T22:12:59.200 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.80813.log 2026-02-22T22:12:59.200 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.70898.log: /var/log/ceph/cluster2-client.mirror.51698.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.51698.log.gz 2026-02-22T22:12:59.200 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose 92.2% -- replaced with /var/log/ceph/cluster2-client.mirror.0.50819.log.gz 2026-02-22T22:12:59.200 INFO:teuthology.orchestra.run.trial026.stderr: -- /var/log/ceph/cluster2-client.mirror.58010.log 2026-02-22T22:12:59.200 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.1.80813.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76924.log 2026-02-22T22:12:59.202 INFO:teuthology.orchestra.run.trial026.stderr: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.70898.log.gz 2026-02-22T22:12:59.202 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.58010.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62532.log 2026-02-22T22:12:59.202 INFO:teuthology.orchestra.run.trial026.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.58010.log.gz 2026-02-22T22:12:59.202 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.76924.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61156.log 2026-02-22T22:12:59.202 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.62532.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.76924.log.gz 2026-02-22T22:12:59.202 INFO:teuthology.orchestra.run.trial026.stderr: 92.5% -- replaced with /var/log/ceph/cluster1-client.mirror.1.80813.log.gz 2026-02-22T22:12:59.202 INFO:teuthology.orchestra.run.trial026.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.62532.log.gz 2026-02-22T22:12:59.203 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61726.log 2026-02-22T22:12:59.203 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75624.log 2026-02-22T22:12:59.203 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.61156.log: 79.7% -- replaced with /var/log/ceph/cluster2-client.mirror.61156.log.gz 2026-02-22T22:12:59.203 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.63424.log 2026-02-22T22:12:59.203 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.61726.log: /var/log/ceph/cluster2-client.mirror.75624.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78993.log 2026-02-22T22:12:59.204 INFO:teuthology.orchestra.run.trial026.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.61726.log.gz 2026-02-22T22:12:59.204 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.63424.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.75624.log.gz 2026-02-22T22:12:59.204 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.82195.log 2026-02-22T22:12:59.205 INFO:teuthology.orchestra.run.trial026.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.63424.log.gz 2026-02-22T22:12:59.205 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81669.log 2026-02-22T22:12:59.205 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.78993.log: /var/log/ceph/cluster1-client.mirror.1.82195.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.72843.log 2026-02-22T22:12:59.205 INFO:teuthology.orchestra.run.trial026.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.78993.log.gz 2026-02-22T22:12:59.205 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.81669.log: 86.8%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68373.log 2026-02-22T22:12:59.205 INFO:teuthology.orchestra.run.trial026.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.81669.log.gz 2026-02-22T22:12:59.205 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.72843.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59546.log 2026-02-22T22:12:59.205 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.68373.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.68373.log.gz 2026-02-22T22:12:59.205 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56587.log 2026-02-22T22:12:59.205 INFO:teuthology.orchestra.run.trial026.stderr: 92.2% -- replaced with /var/log/ceph/cluster1-client.mirror.72843.log.gz 2026-02-22T22:12:59.206 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.59546.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67844.log 2026-02-22T22:12:59.206 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.56587.log: 93.4% -- replaced with /var/log/ceph/cluster1-client.mirror.1.82195.log.gz 2026-02-22T22:12:59.206 INFO:teuthology.orchestra.run.trial026.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.56587.log.gz 2026-02-22T22:12:59.207 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66268.log 2026-02-22T22:12:59.207 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73933.log 2026-02-22T22:12:59.208 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.67844.log: /var/log/ceph/cluster1-client.mirror.66268.log: 88.0% -- replaced with /var/log/ceph/cluster1-client.mirror.66268.log.gz 2026-02-22T22:12:59.208 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.50952.log 2026-02-22T22:12:59.208 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.73933.log: 92.7% -- replaced with /var/log/ceph/cluster2-client.mirror.67844.log.gz 2026-02-22T22:12:59.209 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.83016.log 2026-02-22T22:12:59.209 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.admin.50952.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.50952.log.gz 2026-02-22T22:12:59.209 INFO:teuthology.orchestra.run.trial026.stderr: 92.4% -- replaced with /var/log/ceph/cluster2-client.mirror.73933.log.gz 2026-02-22T22:12:59.210 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66618.log 2026-02-22T22:12:59.210 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78250.log 2026-02-22T22:12:59.210 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.83016.log: /var/log/ceph/cluster2-client.mirror.66618.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.83016.log.gz 2026-02-22T22:12:59.210 INFO:teuthology.orchestra.run.trial026.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.66618.log.gz 2026-02-22T22:12:59.211 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65971.log 2026-02-22T22:12:59.211 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.78250.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82431.log 2026-02-22T22:12:59.212 INFO:teuthology.orchestra.run.trial026.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.78250.log.gz 2026-02-22T22:12:59.212 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.65971.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61502.log 2026-02-22T22:12:59.212 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.82431.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.65971.log.gz 2026-02-22T22:12:59.213 INFO:teuthology.orchestra.run.trial026.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.82431.log.gz 2026-02-22T22:12:59.213 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49085.log 2026-02-22T22:12:59.213 INFO:teuthology.orchestra.run.trial026.stderr:gzip/var/log/ceph/cluster2-client.mirror.61502.log: -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78588.log 2026-02-22T22:12:59.214 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.49085.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.49085.log.gz 2026-02-22T22:12:59.214 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55068.log 2026-02-22T22:12:59.215 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.78588.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.78588.log.gz 2026-02-22T22:12:59.216 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.53253.log 2026-02-22T22:12:59.216 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.55068.log: 89.6% -- replaced with /var/log/ceph/cluster1-client.mirror.55068.log.gz 2026-02-22T22:12:59.217 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55483.log 2026-02-22T22:12:59.217 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.admin.53253.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.admin.53253.log.gz 2026-02-22T22:12:59.218 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67197.log 2026-02-22T22:12:59.221 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.55483.log: 93.6% -- replaced with /var/log/ceph/cluster2-client.mirror.55483.log.gz 2026-02-22T22:12:59.222 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.73064.log 2026-02-22T22:12:59.223 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.67197.log: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.67197.log.gz 2026-02-22T22:12:59.224 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.54401.log 2026-02-22T22:12:59.225 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.73064.log: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.73064.log.gz 2026-02-22T22:12:59.225 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76469.log 2026-02-22T22:12:59.226 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.admin.54401.log: 88.4% -- replaced with /var/log/ceph/cluster1-client.admin.54401.log.gz 2026-02-22T22:12:59.226 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50708.log 2026-02-22T22:12:59.228 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.76469.log: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.76469.log.gz 2026-02-22T22:12:59.229 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.72951.log 2026-02-22T22:12:59.232 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.50708.log: 93.3% -- replaced with /var/log/ceph/cluster1-client.mirror.50708.log.gz 2026-02-22T22:12:59.233 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62420.log 2026-02-22T22:12:59.233 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.72951.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.72951.log.gz 2026-02-22T22:12:59.234 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.74061.log 2026-02-22T22:12:59.235 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.62420.log: 92.7% -- replaced with /var/log/ceph/cluster2-client.mirror.62420.log.gz 2026-02-22T22:12:59.235 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78385.log 2026-02-22T22:12:59.236 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.74061.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.74061.log.gz 2026-02-22T22:12:59.236 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79195.log 2026-02-22T22:12:59.236 INFO:teuthology.orchestra.run.trial026.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.60326.log.gz 2026-02-22T22:12:59.237 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.78385.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.78385.log.gz 2026-02-22T22:12:59.237 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57697.log 2026-02-22T22:12:59.237 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.79195.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59848.log 2026-02-22T22:12:59.237 INFO:teuthology.orchestra.run.trial026.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.79195.log.gz 2026-02-22T22:12:59.238 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.57697.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75149.log 2026-02-22T22:12:59.239 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.59848.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.59848.log.gz 2026-02-22T22:12:59.239 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65104.log 2026-02-22T22:12:59.240 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.75149.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.75149.log.gz 2026-02-22T22:12:59.240 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.47939.log 2026-02-22T22:12:59.241 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.65104.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.65104.log.gz 2026-02-22T22:12:59.241 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74196.log 2026-02-22T22:12:59.242 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.admin.47939.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.47939.log.gz 2026-02-22T22:12:59.242 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74942.log 2026-02-22T22:12:59.243 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.74196.log: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.74196.log.gz 2026-02-22T22:12:59.243 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75975.log 2026-02-22T22:12:59.244 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.74942.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.74942.log.gz 2026-02-22T22:12:59.244 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.62809.log 2026-02-22T22:12:59.246 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.75975.log: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.75975.log.gz 2026-02-22T22:12:59.247 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.63935.log 2026-02-22T22:12:59.247 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.62809.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.62809.log.gz 2026-02-22T22:12:59.248 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75789.log 2026-02-22T22:12:59.249 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.63935.log: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.63935.log.gz 2026-02-22T22:12:59.249 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.83039.log 2026-02-22T22:12:59.251 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.75789.log: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.75789.log.gz 2026-02-22T22:12:59.251 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84115.log 2026-02-22T22:12:59.252 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.83039.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.83039.log.gz 2026-02-22T22:12:59.252 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.71836.log 2026-02-22T22:12:59.253 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.84115.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.84115.log.gz 2026-02-22T22:12:59.253 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74871.log 2026-02-22T22:12:59.254 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.71836.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.71836.log.gz 2026-02-22T22:12:59.254 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.85227.log 2026-02-22T22:12:59.256 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.74871.log: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.74871.log.gz 2026-02-22T22:12:59.257 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.0.82082.log 2026-02-22T22:12:59.257 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.admin.85227.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.85227.log.gz 2026-02-22T22:12:59.257 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.71325.log 2026-02-22T22:12:59.260 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.0.82082.log: 93.3% -- replaced with /var/log/ceph/cluster2-client.mirror.0.82082.log.gz 2026-02-22T22:12:59.261 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71905.log 2026-02-22T22:12:59.261 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.71325.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.71325.log.gz 2026-02-22T22:12:59.262 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67385.log 2026-02-22T22:12:59.263 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.71905.log: 92.7% -- replaced with /var/log/ceph/cluster2-client.mirror.71905.log.gz 2026-02-22T22:12:59.263 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58384.log 2026-02-22T22:12:59.264 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.67385.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.67385.log.gz 2026-02-22T22:12:59.264 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.83722.log 2026-02-22T22:12:59.265 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.58384.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.58384.log.gz 2026-02-22T22:12:59.265 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69929.log 2026-02-22T22:12:59.266 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.83722.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.83722.log.gz 2026-02-22T22:12:59.267 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66172.log 2026-02-22T22:12:59.267 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.69929.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.69929.log.gz 2026-02-22T22:12:59.268 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.83081.log 2026-02-22T22:12:59.268 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.66172.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.66172.log.gz 2026-02-22T22:12:59.269 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.85521.log 2026-02-22T22:12:59.269 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.83081.log: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.61420.log.gz 2026-02-22T22:12:59.269 INFO:teuthology.orchestra.run.trial026.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.83081.log.gz 2026-02-22T22:12:59.270 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75128.log 2026-02-22T22:12:59.270 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.admin.85521.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.83928.log 2026-02-22T22:12:59.270 INFO:teuthology.orchestra.run.trial026.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.85521.log.gz 2026-02-22T22:12:59.270 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.75128.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.75128.log.gz 2026-02-22T22:12:59.271 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82344.log 2026-02-22T22:12:59.271 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.83928.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61543.log 2026-02-22T22:12:59.271 INFO:teuthology.orchestra.run.trial026.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.83928.log.gz 2026-02-22T22:12:59.272 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.82344.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.52866.log 2026-02-22T22:12:59.272 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.61543.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.61543.log.gz 2026-02-22T22:12:59.273 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72598.log 2026-02-22T22:12:59.275 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.3.52866.log: 93.0% -- replaced with /var/log/ceph/cluster1-client.mirror.3.52866.log.gz 2026-02-22T22:12:59.276 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51135.log 2026-02-22T22:12:59.277 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.72598.log: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.72598.log.gz 2026-02-22T22:12:59.277 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69974.log 2026-02-22T22:12:59.278 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.51135.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.51135.log.gz 2026-02-22T22:12:59.278 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83500.log 2026-02-22T22:12:59.280 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.69974.log: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.69974.log.gz 2026-02-22T22:12:59.280 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66526.log 2026-02-22T22:12:59.291 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.83500.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68053.log 2026-02-22T22:12:59.293 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.66526.log: 92.5% -- replaced with /var/log/ceph/cluster1-client.mirror.66526.log.gz 2026-02-22T22:12:59.298 INFO:teuthology.orchestra.run.trial026.stderr: 93.9% -- replaced with /var/log/ceph/cluster2-client.mirror.61748.log.gz 2026-02-22T22:12:59.298 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82983.log 2026-02-22T22:12:59.299 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.68053.log: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.68053.log.gz 2026-02-22T22:12:59.300 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.71036.log 2026-02-22T22:12:59.311 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.82983.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84051.log 2026-02-22T22:12:59.312 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.71036.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.71036.log.gz 2026-02-22T22:12:59.324 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65152.log 2026-02-22T22:12:59.325 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.84051.log: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.84051.log.gz 2026-02-22T22:12:59.335 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60130.log 2026-02-22T22:12:59.336 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.65152.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.65152.log.gz 2026-02-22T22:12:59.358 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.64541.log 2026-02-22T22:12:59.369 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.60130.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60670.log 2026-02-22T22:12:59.370 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.64541.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.64541.log.gz 2026-02-22T22:12:59.381 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66690.log 2026-02-22T22:12:59.382 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.60670.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.60670.log.gz 2026-02-22T22:12:59.390 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60610.log 2026-02-22T22:12:59.400 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.66690.log: 87.7%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77689.log 2026-02-22T22:12:59.403 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.60610.log: -- replaced with /var/log/ceph/cluster1-client.mirror.66690.log.gz 2026-02-22T22:12:59.409 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68866.log 2026-02-22T22:12:59.418 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.77689.log: 88.1%gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.47841.log 2026-02-22T22:12:59.421 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.68866.log: 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.77689.log.gz 2026-02-22T22:12:59.427 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67930.log 2026-02-22T22:12:59.427 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.admin.47841.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.47841.log.gz 2026-02-22T22:12:59.431 INFO:teuthology.orchestra.run.trial026.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.68866.log.gz 2026-02-22T22:12:59.435 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78948.log 2026-02-22T22:12:59.445 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.67930.log: 89.7%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64692.log 2026-02-22T22:12:59.445 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.78948.log: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.78948.log.gz 2026-02-22T22:12:59.449 INFO:teuthology.orchestra.run.trial026.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.67930.log.gz 2026-02-22T22:12:59.453 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.80050.log 2026-02-22T22:12:59.463 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.64692.log: 89.7%gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.64039.log 2026-02-22T22:12:59.463 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.80050.log: 86.6% -- replaced with /var/log/ceph/cluster1-client.mirror.80050.log.gz 2026-02-22T22:12:59.467 INFO:teuthology.orchestra.run.trial026.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.58655.log.gz 2026-02-22T22:12:59.467 INFO:teuthology.orchestra.run.trial026.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.64692.log.gz 2026-02-22T22:12:59.479 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56085.log 2026-02-22T22:12:59.479 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.64039.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.64039.log.gz 2026-02-22T22:12:59.488 INFO:teuthology.orchestra.run.trial026.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.59716.log.gz 2026-02-22T22:12:59.490 INFO:teuthology.orchestra.run.trial026.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.60776.log.gz 2026-02-22T22:12:59.491 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76862.log 2026-02-22T22:12:59.494 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.56085.log: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.56085.log.gz 2026-02-22T22:12:59.496 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77007.log 2026-02-22T22:12:59.496 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.76862.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.76862.log.gz 2026-02-22T22:12:59.496 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69460.log 2026-02-22T22:12:59.499 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.77007.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.77007.log.gz 2026-02-22T22:12:59.500 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.48893.log 2026-02-22T22:12:59.500 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.69460.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.69460.log.gz 2026-02-22T22:12:59.500 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51458.log 2026-02-22T22:12:59.500 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.admin.48893.log: 67.4% -- replaced with /var/log/ceph/cluster2-client.admin.48893.log.gz 2026-02-22T22:12:59.500 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80947.log 2026-02-22T22:12:59.502 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.51458.log: 93.3% -- replaced with /var/log/ceph/cluster2-client.mirror.51458.log.gz 2026-02-22T22:12:59.502 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55698.log 2026-02-22T22:12:59.502 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.80947.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.80947.log.gz 2026-02-22T22:12:59.502 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69330.log 2026-02-22T22:12:59.506 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.55698.log: 93.6% -- replaced with /var/log/ceph/cluster2-client.mirror.55698.log.gz 2026-02-22T22:12:59.506 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58300.log 2026-02-22T22:12:59.507 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.69330.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.69330.log.gz 2026-02-22T22:12:59.507 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.74109.log 2026-02-22T22:12:59.508 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.58300.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.58300.log.gz 2026-02-22T22:12:59.508 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55761.log 2026-02-22T22:12:59.509 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.74109.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.74109.log.gz 2026-02-22T22:12:59.509 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63077.log 2026-02-22T22:12:59.513 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.55761.log: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.55761.log.gz 2026-02-22T22:12:59.513 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81581.log 2026-02-22T22:12:59.514 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.63077.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.63077.log.gz 2026-02-22T22:12:59.514 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54628.log 2026-02-22T22:12:59.515 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.81581.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.81581.log.gz 2026-02-22T22:12:59.516 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78723.log 2026-02-22T22:12:59.516 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.54628.log: 89.4% -- replaced with /var/log/ceph/cluster1-client.mirror.54628.log.gz 2026-02-22T22:12:59.517 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.84854.log 2026-02-22T22:12:59.517 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.78723.log: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.78723.log.gz 2026-02-22T22:12:59.517 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57806.log 2026-02-22T22:12:59.518 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.admin.84854.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.84854.log.gz 2026-02-22T22:12:59.518 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73370.log 2026-02-22T22:12:59.519 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.57806.log: 79.7% -- replaced with /var/log/ceph/cluster2-client.mirror.57806.log.gz 2026-02-22T22:12:59.519 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58406.log 2026-02-22T22:12:59.530 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.73370.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.73691.log 2026-02-22T22:12:59.530 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.58406.log: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.73370.log.gz 2026-02-22T22:12:59.530 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61091.log 2026-02-22T22:12:59.530 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.73691.log: 88.0% -- replaced with /var/log/ceph/cluster1-client.mirror.73691.log.gz 2026-02-22T22:12:59.537 INFO:teuthology.orchestra.run.trial026.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.57236.log.gz 2026-02-22T22:12:59.542 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79555.log 2026-02-22T22:12:59.542 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.61091.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.61091.log.gz 2026-02-22T22:12:59.542 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69953.log 2026-02-22T22:12:59.542 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.79555.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.79555.log.gz 2026-02-22T22:12:59.542 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79825.log 2026-02-22T22:12:59.543 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.69953.log: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.69953.log.gz 2026-02-22T22:12:59.543 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61689.log 2026-02-22T22:12:59.543 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.79825.log: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.79825.log.gz 2026-02-22T22:12:59.543 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.71149.log 2026-02-22T22:12:59.543 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.61689.log: 79.4% -- replaced with /var/log/ceph/cluster2-client.mirror.61689.log.gz 2026-02-22T22:12:59.543 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55675.log 2026-02-22T22:12:59.544 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.71149.log: 92.5% -- replaced with /var/log/ceph/cluster1-client.mirror.71149.log.gz 2026-02-22T22:12:59.545 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68846.log 2026-02-22T22:12:59.547 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.55675.log: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.55675.log.gz 2026-02-22T22:12:59.547 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.71884.log 2026-02-22T22:12:59.550 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.68846.log: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.68846.log.gz 2026-02-22T22:12:59.550 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69484.log 2026-02-22T22:12:59.550 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.71884.log: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.71884.log.gz 2026-02-22T22:12:59.550 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78115.log 2026-02-22T22:12:59.551 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.69484.log: 88.0% -- replaced with /var/log/ceph/cluster1-client.mirror.69484.log.gz 2026-02-22T22:12:59.553 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74963.log 2026-02-22T22:12:59.553 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.78115.log: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.78115.log.gz 2026-02-22T22:12:59.553 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53153.log 2026-02-22T22:12:59.553 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.74963.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.74963.log.gz 2026-02-22T22:12:59.554 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69145.log 2026-02-22T22:12:59.557 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.53153.log: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.53153.log.gz 2026-02-22T22:12:59.557 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58032.log 2026-02-22T22:12:59.557 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.69145.log: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.69145.log.gz 2026-02-22T22:12:59.558 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53306.log 2026-02-22T22:12:59.564 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.58032.log: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.59546.log.gz 2026-02-22T22:12:59.565 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.48150.log 2026-02-22T22:12:59.566 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.53306.log: 91.7% -- replaced with /var/log/ceph/cluster2-client.mirror.53306.log.gz 2026-02-22T22:12:59.566 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72680.log 2026-02-22T22:12:59.569 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.3.48150.log: 93.6% -- replaced with /var/log/ceph/cluster1-client.mirror.3.48150.log.gz 2026-02-22T22:12:59.569 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75087.log 2026-02-22T22:12:59.572 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.72680.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.72680.log.gz 2026-02-22T22:12:59.572 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73790.log 2026-02-22T22:12:59.572 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.75087.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.75087.log.gz 2026-02-22T22:12:59.572 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.47155.log 2026-02-22T22:12:59.572 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.73790.log: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.73790.log.gz 2026-02-22T22:12:59.575 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79353.log 2026-02-22T22:12:59.575 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.admin.47155.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.47155.log.gz 2026-02-22T22:12:59.575 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62276.log 2026-02-22T22:12:59.575 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.79353.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.79353.log.gz 2026-02-22T22:12:59.576 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70219.log 2026-02-22T22:12:59.576 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.62276.log: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.62276.log.gz 2026-02-22T22:12:59.576 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.61261.log 2026-02-22T22:12:59.577 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.70219.log: 92.5% -- replaced with /var/log/ceph/cluster1-client.mirror.70219.log.gz 2026-02-22T22:12:59.577 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58167.log 2026-02-22T22:12:59.582 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.2.61261.log: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.58487.log.gz 2026-02-22T22:12:59.583 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55460.log 2026-02-22T22:12:59.584 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.58167.log: 90.3% -- replaced with /var/log/ceph/cluster1-client.mirror.58167.log.gz 2026-02-22T22:12:59.584 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.46913.log 2026-02-22T22:12:59.587 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.55460.log: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.55460.log.gz 2026-02-22T22:12:59.589 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76162.log 2026-02-22T22:12:59.589 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.admin.46913.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.46913.log.gz 2026-02-22T22:12:59.589 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79938.log 2026-02-22T22:12:59.592 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.76162.log: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.76162.log.gz 2026-02-22T22:12:59.592 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84013.log 2026-02-22T22:12:59.592 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.79938.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.79938.log.gz 2026-02-22T22:12:59.592 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53065.log 2026-02-22T22:12:59.594 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.84013.log: 91.3% -- replaced with /var/log/ceph/cluster2-client.mirror.84013.log.gz 2026-02-22T22:12:59.594 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55956.log 2026-02-22T22:12:59.594 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.53065.log: 88.8% -- replaced with /var/log/ceph/cluster1-client.mirror.53065.log.gz 2026-02-22T22:12:59.595 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54164.log 2026-02-22T22:12:59.599 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.55956.log: 93.6% -- replaced with /var/log/ceph/cluster2-client.mirror.55956.log.gz 2026-02-22T22:12:59.599 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.82705.log 2026-02-22T22:12:59.601 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.54164.log: 89.1% -- replaced with /var/log/ceph/cluster1-client.mirror.54164.log.gz 2026-02-22T22:12:59.602 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69547.log 2026-02-22T22:12:59.602 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.82705.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.82705.log.gz 2026-02-22T22:12:59.602 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59831.log 2026-02-22T22:12:59.602 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.69547.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.69547.log.gz 2026-02-22T22:12:59.605 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61526.log 2026-02-22T22:12:59.605 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.59831.log: 79.5% -- replaced with /var/log/ceph/cluster2-client.mirror.59831.log.gz 2026-02-22T22:12:59.605 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.72505.log 2026-02-22T22:12:59.606 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.61526.log: 79.7% -- replaced with /var/log/ceph/cluster2-client.mirror.61526.log.gz 2026-02-22T22:12:59.606 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75479.log 2026-02-22T22:12:59.606 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.72505.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.72505.log.gz 2026-02-22T22:12:59.606 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57638.log 2026-02-22T22:12:59.609 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.75479.log: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.75479.log.gz 2026-02-22T22:12:59.609 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78880.log 2026-02-22T22:12:59.609 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.57638.log: 79.8% -- replaced with /var/log/ceph/cluster2-client.mirror.57638.log.gz 2026-02-22T22:12:59.609 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67799.log 2026-02-22T22:12:59.609 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.78880.log: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.78880.log.gz 2026-02-22T22:12:59.609 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81953.log 2026-02-22T22:12:59.609 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.67799.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.67799.log.gz 2026-02-22T22:12:59.612 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79960.log 2026-02-22T22:12:59.612 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.81953.log: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.81953.log.gz 2026-02-22T22:12:59.612 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63326.log 2026-02-22T22:12:59.612 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.79960.log: 86.5% -- replaced with /var/log/ceph/cluster1-client.mirror.79960.log.gz 2026-02-22T22:12:59.612 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79105.log 2026-02-22T22:12:59.613 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.63326.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.63326.log.gz 2026-02-22T22:12:59.614 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59332.log 2026-02-22T22:12:59.614 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.79105.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.79105.log.gz 2026-02-22T22:12:59.614 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66084.log 2026-02-22T22:12:59.614 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.59332.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.59332.log.gz 2026-02-22T22:12:59.615 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.51388.log 2026-02-22T22:12:59.616 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.66084.log: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.66084.log.gz 2026-02-22T22:12:59.616 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82285.log 2026-02-22T22:12:59.617 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.admin.51388.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.51388.log.gz 2026-02-22T22:12:59.618 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69994.log 2026-02-22T22:12:59.618 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.82285.log: 79.8% -- replaced with /var/log/ceph/cluster2-client.mirror.82285.log.gz 2026-02-22T22:12:59.619 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.71704.log 2026-02-22T22:12:59.620 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.69994.log: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.69994.log.gz 2026-02-22T22:12:59.620 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59796.log 2026-02-22T22:12:59.621 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.71704.log: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.71704.log.gz 2026-02-22T22:12:59.622 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79420.log 2026-02-22T22:12:59.631 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.59796.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65357.log 2026-02-22T22:12:59.644 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.79420.log: 86.9%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68399.log 2026-02-22T22:12:59.644 INFO:teuthology.orchestra.run.trial026.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.79420.log.gz 2026-02-22T22:12:59.644 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.65357.log: 94.9% -- replaced with /var/log/ceph/cluster1-client.mirror.65357.log.gz 2026-02-22T22:12:59.650 INFO:teuthology.orchestra.run.trial026.stderr: 93.9% -- replaced with /var/log/ceph/cluster2-client.mirror.57865.log.gz 2026-02-22T22:12:59.650 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64399.log 2026-02-22T22:12:59.650 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.68399.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.68399.log.gz 2026-02-22T22:12:59.651 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83246.log 2026-02-22T22:12:59.651 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.64399.log: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.64399.log.gz 2026-02-22T22:12:59.652 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66666.log 2026-02-22T22:12:59.660 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.83246.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.63199.log 2026-02-22T22:12:59.661 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.66666.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.66666.log.gz 2026-02-22T22:12:59.669 INFO:teuthology.orchestra.run.trial026.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.83500.log.gz 2026-02-22T22:12:59.669 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.64766.log 2026-02-22T22:12:59.670 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.63199.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.63199.log.gz 2026-02-22T22:12:59.670 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59057.log 2026-02-22T22:12:59.671 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.64766.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.64766.log.gz 2026-02-22T22:12:59.671 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.0.65357.log 2026-02-22T22:12:59.672 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.59057.log: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.59057.log.gz 2026-02-22T22:12:59.673 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61771.log 2026-02-22T22:12:59.673 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.0.65357.log: 93.9% -- replaced with /var/log/ceph/cluster2-client.mirror.57604.log.gz 2026-02-22T22:12:59.674 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53850.log 2026-02-22T22:12:59.674 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.61771.log: 79.6% -- replaced with /var/log/ceph/cluster2-client.mirror.61771.log.gz 2026-02-22T22:12:59.675 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52202.log 2026-02-22T22:12:59.680 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.53850.log: 94.1% -- replaced with /var/log/ceph/cluster2-client.mirror.0.65357.log.gz 2026-02-22T22:12:59.680 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82770.log 2026-02-22T22:12:59.681 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.52202.log: 88.0% -- replaced with /var/log/ceph/cluster1-client.mirror.52202.log.gz 2026-02-22T22:12:59.681 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59754.log 2026-02-22T22:12:59.682 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.82770.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.82770.log.gz 2026-02-22T22:12:59.682 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65843.log 2026-02-22T22:12:59.683 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.59754.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.59754.log.gz 2026-02-22T22:12:59.684 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84138.log 2026-02-22T22:12:59.685 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.65843.log: 92.7% -- replaced with /var/log/ceph/cluster2-client.mirror.65843.log.gz 2026-02-22T22:12:59.686 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.72802.log 2026-02-22T22:12:59.686 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.84138.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.84138.log.gz 2026-02-22T22:12:59.687 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57889.log 2026-02-22T22:12:59.687 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.72802.log: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.72802.log.gz 2026-02-22T22:12:59.688 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66548.log 2026-02-22T22:12:59.688 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.57889.log: 79.5% -- replaced with /var/log/ceph/cluster2-client.mirror.57889.log.gz 2026-02-22T22:12:59.689 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62133.log 2026-02-22T22:12:59.689 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.66548.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.66548.log.gz 2026-02-22T22:12:59.690 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.74278.log 2026-02-22T22:12:59.690 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.62133.log: 1.3% -- replaced with /var/log/ceph/cluster2-client.mirror.62133.log.gz 2026-02-22T22:12:59.691 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.84830.log 2026-02-22T22:12:59.691 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.74278.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.74278.log.gz 2026-02-22T22:12:59.692 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.53850.log 2026-02-22T22:12:59.692 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.admin.84830.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.84830.log.gz 2026-02-22T22:12:59.692 INFO:teuthology.orchestra.run.trial026.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.53850.log.gz 2026-02-22T22:12:59.693 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75211.log 2026-02-22T22:12:59.693 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.3.48293.log 2026-02-22T22:12:59.693 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.0.53850.log: /var/log/ceph/cluster2-client.mirror.75211.log: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.62107.log.gz 2026-02-22T22:12:59.694 INFO:teuthology.orchestra.run.trial026.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.75211.log.gz 2026-02-22T22:12:59.694 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56687.log 2026-02-22T22:12:59.694 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.3.48293.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.47792.log 2026-02-22T22:12:59.695 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.56687.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.56687.log.gz 2026-02-22T22:12:59.695 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75314.log 2026-02-22T22:12:59.695 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.admin.47792.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.47792.log.gz 2026-02-22T22:12:59.696 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54828.log 2026-02-22T22:12:59.696 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.75314.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.75314.log.gz 2026-02-22T22:12:59.697 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.50463.log 2026-02-22T22:12:59.702 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.54828.log: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.57697.log.gz 2026-02-22T22:12:59.703 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70648.log 2026-02-22T22:12:59.704 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.admin.50463.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.50463.log.gz 2026-02-22T22:12:59.704 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.85630.log 2026-02-22T22:12:59.704 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.70648.log: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.70648.log.gz 2026-02-22T22:12:59.706 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76655.log 2026-02-22T22:12:59.706 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.admin.85630.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.85630.log.gz 2026-02-22T22:12:59.706 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.83837.log 2026-02-22T22:12:59.707 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.76655.log: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.76655.log.gz 2026-02-22T22:12:59.710 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.85276.log 2026-02-22T22:12:59.710 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.83837.log: 79.8% -- replaced with /var/log/ceph/cluster1-client.mirror.83837.log.gz 2026-02-22T22:12:59.710 INFO:teuthology.orchestra.run.trial026.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.3.48293.log.gz 2026-02-22T22:12:59.710 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83609.log 2026-02-22T22:12:59.710 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.admin.85276.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52679.log 2026-02-22T22:12:59.710 INFO:teuthology.orchestra.run.trial026.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.85276.log.gz 2026-02-22T22:12:59.710 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.83609.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66994.log 2026-02-22T22:12:59.710 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.52679.log: 84.6% -- replaced with /var/log/ceph/cluster2-client.mirror.83609.log.gz 2026-02-22T22:12:59.710 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56261.log 2026-02-22T22:12:59.712 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.66994.log: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.66994.log.gz 2026-02-22T22:12:59.712 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.0.58167.log 2026-02-22T22:12:59.712 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.56261.log: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.52679.log.gz 2026-02-22T22:12:59.713 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57390.log 2026-02-22T22:12:59.715 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.0.58167.log: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.56261.log.gz 2026-02-22T22:12:59.715 INFO:teuthology.orchestra.run.trial026.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.61668.log.gz 2026-02-22T22:12:59.715 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59694.log 2026-02-22T22:12:59.716 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.57390.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64174.log 2026-02-22T22:12:59.716 INFO:teuthology.orchestra.run.trial026.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.57390.log.gz 2026-02-22T22:12:59.716 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.59694.log: 92.8% -- replaced with /var/log/ceph/cluster2-client.mirror.0.58167.log.gz 2026-02-22T22:12:59.729 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.80140.log 2026-02-22T22:12:59.729 INFO:teuthology.orchestra.run.trial026.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.59694.log.gz 2026-02-22T22:12:59.729 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.64174.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58767.log 2026-02-22T22:12:59.729 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.80140.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.53753.log 2026-02-22T22:12:59.729 INFO:teuthology.orchestra.run.trial026.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.64174.log.gz 2026-02-22T22:12:59.729 INFO:teuthology.orchestra.run.trial026.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.80140.log.gz 2026-02-22T22:12:59.729 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.58767.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75872.log 2026-02-22T22:12:59.729 INFO:teuthology.orchestra.run.trial026.stderr: 79.8% -- replaced with /var/log/ceph/cluster2-client.mirror.58767.log.gz 2026-02-22T22:12:59.729 INFO:teuthology.orchestra.run.trial026.stderr: 93.9% -- replaced with /var/log/ceph/cluster2-client.mirror.59459.log.gz 2026-02-22T22:12:59.729 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.admin.53753.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65921.log 2026-02-22T22:12:59.729 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.75872.log: 87.9% -- replaced with /var/log/ceph/cluster1-client.admin.53753.log.gz 2026-02-22T22:12:59.729 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77523.log 2026-02-22T22:12:59.729 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.65921.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51037.log 2026-02-22T22:12:59.729 INFO:teuthology.orchestra.run.trial026.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.75872.log.gz 2026-02-22T22:12:59.730 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60088.log 2026-02-22T22:12:59.730 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.77523.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.65921.log.gz 2026-02-22T22:12:59.730 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.51037.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69412.log 2026-02-22T22:12:59.730 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.60088.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60939.log 2026-02-22T22:12:59.730 INFO:teuthology.orchestra.run.trial026.stderr: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.51037.log.gz 2026-02-22T22:12:59.730 INFO:teuthology.orchestra.run.trial026.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.60130.log.gz 2026-02-22T22:12:59.730 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.69412.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.60088.log.gz 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.69412.log.gz 2026-02-22T22:12:59.730 INFO:teuthology.orchestra.run.trial026.stderr: 2026-02-22T22:12:59.730 INFO:teuthology.orchestra.run.trial026.stderr:gzip 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.77523.log.gz 2026-02-22T22:12:59.730 INFO:teuthology.orchestra.run.trial026.stderr: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66331.log 2026-02-22T22:12:59.730 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.60939.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69243.log 2026-02-22T22:12:59.730 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.66331.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62065.log 2026-02-22T22:12:59.730 INFO:teuthology.orchestra.run.trial026.stderr: 92.1% -- replaced with /var/log/ceph/cluster1-client.mirror.2.61261.log.gz 2026-02-22T22:12:59.730 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.69243.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.66331.log.gz 2026-02-22T22:12:59.730 INFO:teuthology.orchestra.run.trial026.stderr:gzip 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.69243.log.gz 2026-02-22T22:12:59.730 INFO:teuthology.orchestra.run.trial026.stderr: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80312.log 2026-02-22T22:12:59.730 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.62065.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83458.log 2026-02-22T22:12:59.730 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.80312.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59160.log 2026-02-22T22:12:59.730 INFO:teuthology.orchestra.run.trial026.stderr: 79.7% -- replaced with /var/log/ceph/cluster2-client.mirror.80312.log.gz 2026-02-22T22:12:59.730 INFO:teuthology.orchestra.run.trial026.stderr: 89.6%/var/log/ceph/cluster2-client.mirror.83458.log: -- replaced with /var/log/ceph/cluster2-client.mirror.62065.log.gzgzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78858.log 2026-02-22T22:12:59.731 INFO:teuthology.orchestra.run.trial026.stderr: 2026-02-22T22:12:59.731 INFO:teuthology.orchestra.run.trial026.stderr:gzip/var/log/ceph/cluster2-client.mirror.59160.log: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84323.log 2026-02-22T22:12:59.731 INFO:teuthology.orchestra.run.trial026.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.60051.log.gz 2026-02-22T22:12:59.731 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.78858.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.59160.log.gz 2026-02-22T22:12:59.731 INFO:teuthology.orchestra.run.trial026.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.83458.log.gz 2026-02-22T22:12:59.731 INFO:teuthology.orchestra.run.trial026.stderr: 86.6% -- replaced with /var/log/ceph/cluster1-client.mirror.78858.log.gz 2026-02-22T22:12:59.731 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75025.log 2026-02-22T22:12:59.731 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.84323.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77751.log 2026-02-22T22:12:59.731 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.75025.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.64565.log 2026-02-22T22:12:59.731 INFO:teuthology.orchestra.run.trial026.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.84323.log.gz 2026-02-22T22:12:59.731 INFO:teuthology.orchestra.run.trial026.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.75025.log.gz/var/log/ceph/cluster2-client.mirror.77751.log: 2026-02-22T22:12:59.731 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79465.log 2026-02-22T22:12:59.731 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.64565.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.77751.log.gz 2026-02-22T22:12:59.731 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80968.log 2026-02-22T22:12:59.731 INFO:teuthology.orchestra.run.trial026.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.64565.log.gz 2026-02-22T22:12:59.731 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76340.log 2026-02-22T22:12:59.731 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.79465.log: /var/log/ceph/cluster2-client.mirror.80968.log: 86.6% -- replaced with /var/log/ceph/cluster1-client.mirror.79465.log.gzgzip 2026-02-22T22:12:59.731 INFO:teuthology.orchestra.run.trial026.stderr: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64132.log 2026-02-22T22:12:59.731 INFO:teuthology.orchestra.run.trial026.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.80968.log.gz 2026-02-22T22:12:59.731 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.76340.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81932.log 2026-02-22T22:12:59.731 INFO:teuthology.orchestra.run.trial026.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.76340.log.gz 2026-02-22T22:12:59.731 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.51361.log 2026-02-22T22:12:59.731 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.64132.log: /var/log/ceph/cluster2-client.mirror.81932.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.80185.log 2026-02-22T22:12:59.732 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.admin.51361.log: gzip -5 --verbose 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.81932.log.gz 2026-02-22T22:12:59.732 INFO:teuthology.orchestra.run.trial026.stderr: -- /var/log/ceph/cluster2-client.mirror.69388.log 2026-02-22T22:12:59.732 INFO:teuthology.orchestra.run.trial026.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.51361.log.gz 2026-02-22T22:12:59.732 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.80185.log: gzip -5 --verbose -- 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.80185.log.gz 2026-02-22T22:12:59.732 INFO:teuthology.orchestra.run.trial026.stderr: /var/log/ceph/cluster2-client.mirror.80329.log 2026-02-22T22:12:59.732 INFO:teuthology.orchestra.run.trial026.stderr: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.64132.log.gz 2026-02-22T22:12:59.732 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.69388.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64214.log 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.69388.log.gz 2026-02-22T22:12:59.732 INFO:teuthology.orchestra.run.trial026.stderr: 2026-02-22T22:12:59.732 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.80329.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67448.log 2026-02-22T22:12:59.732 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.64214.log: gzip 89.6% -5 --verbose -- /var/log/ceph/cluster1-client.mirror.80095.log 2026-02-22T22:12:59.732 INFO:teuthology.orchestra.run.trial026.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.80329.log.gz 2026-02-22T22:12:59.732 INFO:teuthology.orchestra.run.trial026.stderr: 90.0% -- replaced with /var/log/ceph/cluster2-client.mirror.64214.log.gz 2026-02-22T22:12:59.732 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.67448.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60917.log 2026-02-22T22:12:59.732 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.80095.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55827.log 2026-02-22T22:12:59.732 INFO:teuthology.orchestra.run.trial026.stderr: 86.8%/var/log/ceph/cluster2-client.mirror.60917.log: -- replaced with /var/log/ceph/cluster1-client.mirror.80095.log.gz 2026-02-22T22:12:59.732 INFO:teuthology.orchestra.run.trial026.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.67448.log.gz 2026-02-22T22:12:59.732 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61480.log 2026-02-22T22:12:59.732 INFO:teuthology.orchestra.run.trial026.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.60917.log.gz 2026-02-22T22:12:59.732 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.55827.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52330.log 2026-02-22T22:12:59.737 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.61480.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76800.log 2026-02-22T22:12:59.737 INFO:teuthology.orchestra.run.trial026.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.61480.log.gz 2026-02-22T22:12:59.737 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.52330.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54996.log 2026-02-22T22:12:59.737 INFO:teuthology.orchestra.run.trial026.stderr:gzip/var/log/ceph/cluster2-client.mirror.76800.log: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84034.log 2026-02-22T22:12:59.737 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.54996.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.76800.log.gz 2026-02-22T22:12:59.737 INFO:teuthology.orchestra.run.trial026.stderr: 89.3%gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70853.log 2026-02-22T22:12:59.737 INFO:teuthology.orchestra.run.trial026.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.54996.log.gz 2026-02-22T22:12:59.737 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.84034.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70130.log 2026-02-22T22:12:59.737 INFO:teuthology.orchestra.run.trial026.stderr: 79.7%gzip -- replaced with /var/log/ceph/cluster2-client.mirror.84034.log.gz 2026-02-22T22:12:59.737 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.70853.log: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70781.log 2026-02-22T22:12:59.738 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.70130.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.70853.log.gz 2026-02-22T22:12:59.738 INFO:teuthology.orchestra.run.trial026.stderr:gzip 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.70130.log.gz 2026-02-22T22:12:59.738 INFO:teuthology.orchestra.run.trial026.stderr: -5 --verbose -- /var/log/ceph/cluster2-client.admin.85080.log 2026-02-22T22:12:59.738 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.70781.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.1.61877.log 2026-02-22T22:12:59.738 INFO:teuthology.orchestra.run.trial026.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.70781.log.gz/var/log/ceph/cluster2-client.admin.85080.log: 2026-02-22T22:12:59.738 INFO:teuthology.orchestra.run.trial026.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.85080.log.gz 2026-02-22T22:12:59.738 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78768.log 93.5% 2026-02-22T22:12:59.738 INFO:teuthology.orchestra.run.trial026.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.55827.log.gz 2026-02-22T22:12:59.738 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.1.61877.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.43538.log 2026-02-22T22:12:59.738 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.78768.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55133.log 2026-02-22T22:12:59.738 INFO:teuthology.orchestra.run.trial026.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.78768.log.gz 2026-02-22T22:12:59.738 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.admin.43538.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.43538.log.gz 2026-02-22T22:12:59.738 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78790.log 2026-02-22T22:12:59.738 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.55133.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77482.log 2026-02-22T22:12:59.738 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.78790.log: gzip -5 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.78790.log.gz 2026-02-22T22:12:59.738 INFO:teuthology.orchestra.run.trial026.stderr: --verbose -- /var/log/ceph/cluster1-client.mirror.67823.log 2026-02-22T22:12:59.738 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.77482.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.62669.log 2026-02-22T22:12:59.738 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.67823.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.77482.log.gz 2026-02-22T22:12:59.738 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57456.log 2026-02-22T22:12:59.738 INFO:teuthology.orchestra.run.trial026.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.67823.log.gz 2026-02-22T22:12:59.738 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.57456.log 2026-02-22T22:12:59.738 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.62669.log: /var/log/ceph/cluster2-client.mirror.57456.log: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.62669.log.gz 2026-02-22T22:12:59.738 INFO:teuthology.orchestra.run.trial026.stderr: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.55133.log.gz 2026-02-22T22:12:59.740 INFO:teuthology.orchestra.run.trial026.stderr: 92.8% -- replaced with /var/log/ceph/cluster1-client.mirror.0.53850.log.gz 2026-02-22T22:12:59.741 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.64264.log 2026-02-22T22:12:59.741 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.2.57456.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.3.60178.log 2026-02-22T22:12:59.742 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.64264.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79623.log 2026-02-22T22:12:59.742 INFO:teuthology.orchestra.run.trial026.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.64264.log.gz 2026-02-22T22:12:59.742 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.3.60178.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.82410.log 2026-02-22T22:12:59.742 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose/var/log/ceph/cluster1-client.mirror.79623.log: -- /var/log/ceph/cluster1-client.mirror.78205.log 2026-02-22T22:12:59.742 INFO:teuthology.orchestra.run.trial026.stderr: 86.6% -- replaced with /var/log/ceph/cluster1-client.mirror.79623.log.gz 2026-02-22T22:12:59.743 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.82410.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54296.log 2026-02-22T22:12:59.743 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.78205.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.82410.log.gz 2026-02-22T22:12:59.743 INFO:teuthology.orchestra.run.trial026.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.78205.log.gz 2026-02-22T22:12:59.743 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58096.log 2026-02-22T22:12:59.743 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66413.log 2026-02-22T22:12:59.744 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.54296.log: /var/log/ceph/cluster2-client.mirror.58096.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.58096.log.gz 2026-02-22T22:12:59.744 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68005.log 2026-02-22T22:12:59.745 INFO:teuthology.orchestra.run.trial026.stderr: 94.0% -- replaced with /var/log/ceph/cluster2-client.mirror.57456.log.gz 2026-02-22T22:12:59.745 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.66413.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.66413.log.gz 2026-02-22T22:12:59.745 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78138.log 2026-02-22T22:12:59.745 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.68005.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50819.log 2026-02-22T22:12:59.745 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.78138.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.68005.log.gz 2026-02-22T22:12:59.746 INFO:teuthology.orchestra.run.trial026.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.78138.log.gz 2026-02-22T22:12:59.746 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71788.log 2026-02-22T22:12:59.746 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.50819.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.83358.log 2026-02-22T22:12:59.746 INFO:teuthology.orchestra.run.trial026.stderr: 84.0% -- replaced with /var/log/ceph/cluster1-client.mirror.50819.log.gz 2026-02-22T22:12:59.747 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.71788.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.64063.log 2026-02-22T22:12:59.747 INFO:teuthology.orchestra.run.trial026.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.71788.log.gz 2026-02-22T22:12:59.747 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.83358.log: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.83358.log.gz 2026-02-22T22:12:59.747 INFO:teuthology.orchestra.run.trial026.stderr: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.54296.log.gz 2026-02-22T22:12:59.748 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76759.log 2026-02-22T22:12:59.748 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.63894.log 2026-02-22T22:12:59.748 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.64063.log: /var/log/ceph/cluster2-client.mirror.76759.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67313.log 2026-02-22T22:12:59.749 INFO:teuthology.orchestra.run.trial026.stderr: 88.1% 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.64063.log.gz 2026-02-22T22:12:59.749 INFO:teuthology.orchestra.run.trial026.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.76759.log.gz 2026-02-22T22:12:59.750 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.63894.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77937.log 2026-02-22T22:12:59.750 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.67313.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.63894.log.gz 2026-02-22T22:12:59.750 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.47498.log 2026-02-22T22:12:59.750 INFO:teuthology.orchestra.run.trial026.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.67313.log.gz 2026-02-22T22:12:59.750 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.77937.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77358.log 2026-02-22T22:12:59.750 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.admin.47498.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79263.log 2026-02-22T22:12:59.750 INFO:teuthology.orchestra.run.trial026.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.47498.log.gz 2026-02-22T22:12:59.750 INFO:teuthology.orchestra.run.trial026.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.77937.log.gz 2026-02-22T22:12:59.751 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.77358.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.71812.log 2026-02-22T22:12:59.751 INFO:teuthology.orchestra.run.trial026.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.77358.log.gz 2026-02-22T22:12:59.751 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.79263.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.62857.log 2026-02-22T22:12:59.751 INFO:teuthology.orchestra.run.trial026.stderr: 86.6% -- replaced with /var/log/ceph/cluster1-client.mirror.79263.log.gz 2026-02-22T22:12:59.752 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77606.log 2026-02-22T22:12:59.752 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.71812.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81911.log 2026-02-22T22:12:59.752 INFO:teuthology.orchestra.run.trial026.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.71812.log.gz 2026-02-22T22:12:59.752 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.62857.log: /var/log/ceph/cluster2-client.mirror.77606.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.62857.log.gz 2026-02-22T22:12:59.753 INFO:teuthology.orchestra.run.trial026.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.77606.log.gz 2026-02-22T22:12:59.753 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.46714.log 2026-02-22T22:12:59.753 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68423.log 2026-02-22T22:12:59.753 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.81911.log: /var/log/ceph/cluster2-client.admin.46714.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52266.log 2026-02-22T22:12:59.753 INFO:teuthology.orchestra.run.trial026.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.46714.log.gz 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.81911.log.gz 2026-02-22T22:12:59.753 INFO:teuthology.orchestra.run.trial026.stderr: 2026-02-22T22:12:59.754 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.68423.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53575.log 2026-02-22T22:12:59.754 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.52266.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68671.log 2026-02-22T22:12:59.754 INFO:teuthology.orchestra.run.trial026.stderr: 87.5%/var/log/ceph/cluster1-client.mirror.53575.log: -- replaced with /var/log/ceph/cluster1-client.mirror.68423.log.gz 2026-02-22T22:12:59.755 INFO:teuthology.orchestra.run.trial026.stderr: 89.1% -- replaced with /var/log/ceph/cluster1-client.mirror.53575.log.gz 2026-02-22T22:12:59.755 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83314.log 2026-02-22T22:12:59.755 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.68671.log: 94.0% -- replaced with /var/log/ceph/cluster2-client.mirror.3.60178.log.gz 2026-02-22T22:12:59.756 INFO:teuthology.orchestra.run.trial026.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.68671.log.gz 2026-02-22T22:12:59.757 INFO:teuthology.orchestra.run.trial026.stderr: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.52266.log.gz 2026-02-22T22:12:59.758 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67426.log 2026-02-22T22:12:59.759 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.83314.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.71366.log 2026-02-22T22:12:59.759 INFO:teuthology.orchestra.run.trial026.stderr: 88.8%/var/log/ceph/cluster1-client.mirror.67426.log: -- replaced with /var/log/ceph/cluster2-client.mirror.83314.log.gz 2026-02-22T22:12:59.760 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52308.log 2026-02-22T22:12:59.760 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.71366.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54910.log 2026-02-22T22:12:59.760 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.52308.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56868.log 2026-02-22T22:12:59.760 INFO:teuthology.orchestra.run.trial026.stderr: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.67426.log.gz 2026-02-22T22:12:59.761 INFO:teuthology.orchestra.run.trial026.stderr: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.52308.log.gz 2026-02-22T22:12:59.761 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.54910.log: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.71366.log.gz 2026-02-22T22:12:59.761 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81752.log 2026-02-22T22:12:59.761 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.56868.log: 93.0% -- replaced with /var/log/ceph/cluster1-client.mirror.2.57456.log.gz 2026-02-22T22:12:59.761 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.80208.log 2026-02-22T22:12:59.761 INFO:teuthology.orchestra.run.trial026.stderr: 94.3% 86.9% -- replaced with /var/log/ceph/cluster2-client.mirror.1.61877.log.gz 2026-02-22T22:12:59.761 INFO:teuthology.orchestra.run.trial026.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.56868.log.gz 2026-02-22T22:12:59.762 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70624.log 2026-02-22T22:12:59.762 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.81752.log: /var/log/ceph/cluster1-client.mirror.80208.log: 91.7% 79.7% -- replaced with /var/log/ceph/cluster2-client.mirror.81752.log.gz 2026-02-22T22:12:59.762 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77193.log 2026-02-22T22:12:59.762 INFO:teuthology.orchestra.run.trial026.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.54910.log.gz 2026-02-22T22:12:59.762 INFO:teuthology.orchestra.run.trial026.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.80208.log.gz 2026-02-22T22:12:59.762 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.70624.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69195.log 2026-02-22T22:12:59.762 INFO:teuthology.orchestra.run.trial026.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.70624.log.gz 2026-02-22T22:12:59.762 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.77193.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77317.log 2026-02-22T22:12:59.764 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83060.log 2026-02-22T22:12:59.764 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.69195.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.77193.log.gz 2026-02-22T22:12:59.764 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.77317.log: gzip -5 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.69195.log.gz 2026-02-22T22:12:59.764 INFO:teuthology.orchestra.run.trial026.stderr: --verbose -- /var/log/ceph/cluster2-client.mirror.74228.log 2026-02-22T22:12:59.764 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.83060.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.77317.log.gz 2026-02-22T22:12:59.764 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52130.log 2026-02-22T22:12:59.764 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.74228.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.83060.log.gz 2026-02-22T22:12:59.764 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.72154.log 2026-02-22T22:12:59.764 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.52130.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60650.log 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.74228.log.gz 2026-02-22T22:12:59.764 INFO:teuthology.orchestra.run.trial026.stderr: 2026-02-22T22:12:59.764 INFO:teuthology.orchestra.run.trial026.stderr: 88.4% -- replaced with /var/log/ceph/cluster1-client.mirror.52130.log.gz 2026-02-22T22:12:59.764 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.72154.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67510.log 2026-02-22T22:12:59.766 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.60650.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56367.log 2026-02-22T22:12:59.766 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.67510.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51859.log 2026-02-22T22:12:59.766 INFO:teuthology.orchestra.run.trial026.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.67510.log.gz 2026-02-22T22:12:59.766 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.56367.log: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.60650.log.gz 2026-02-22T22:12:59.766 INFO:teuthology.orchestra.run.trial026.stderr: 92.5% -- replaced with /var/log/ceph/cluster1-client.mirror.72154.log.gz 2026-02-22T22:12:59.766 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76945.log 2026-02-22T22:12:59.766 INFO:teuthology.orchestra.run.trial026.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.56367.log.gz 2026-02-22T22:12:59.766 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74726.log 2026-02-22T22:12:59.766 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.51859.log: /var/log/ceph/cluster2-client.mirror.76945.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63989.log 2026-02-22T22:12:59.766 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.74726.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.76945.log.gz 2026-02-22T22:12:59.766 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72377.log 2026-02-22T22:12:59.766 INFO:teuthology.orchestra.run.trial026.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.74726.log.gz 2026-02-22T22:12:59.766 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.63989.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69866.log 2026-02-22T22:12:59.767 INFO:teuthology.orchestra.run.trial026.stderr:gzip/var/log/ceph/cluster2-client.mirror.72377.log: -5 --verbose -- /var/log/ceph/cluster2-client.admin.43984.log 2026-02-22T22:12:59.767 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.69866.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.63989.log.gz 2026-02-22T22:12:59.767 INFO:teuthology.orchestra.run.trial026.stderr:gzip 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.69866.log.gz 2026-02-22T22:12:59.767 INFO:teuthology.orchestra.run.trial026.stderr: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57973.log 2026-02-22T22:12:59.767 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.admin.43984.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80988.log 2026-02-22T22:12:59.767 INFO:teuthology.orchestra.run.trial026.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.43984.log.gz 2026-02-22T22:12:59.769 INFO:teuthology.orchestra.run.trial026.stderr: 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.72377.log.gz/var/log/ceph/cluster2-client.mirror.57973.log: 2026-02-22T22:12:59.769 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59504.log 2026-02-22T22:12:59.769 INFO:teuthology.orchestra.run.trial026.stderr: 79.7% -- replaced with /var/log/ceph/cluster2-client.mirror.57973.log.gz 2026-02-22T22:12:59.769 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.80988.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75851.log 2026-02-22T22:12:59.769 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.59504.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65750.log 2026-02-22T22:12:59.769 INFO:teuthology.orchestra.run.trial026.stderr: 93.5%gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51571.log 2026-02-22T22:12:59.769 INFO:teuthology.orchestra.run.trial026.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.51859.log.gz 2026-02-22T22:12:59.769 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.75851.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.59504.log.gz/var/log/ceph/cluster1-client.mirror.65750.log: 2026-02-22T22:12:59.769 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65193.log 2026-02-22T22:12:59.770 INFO:teuthology.orchestra.run.trial026.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.65750.log.gz 2026-02-22T22:12:59.770 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.51571.log: 76.9% -- replaced with /var/log/ceph/cluster1-client.mirror.51571.log.gzgzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59140.log 2026-02-22T22:12:59.770 INFO:teuthology.orchestra.run.trial026.stderr: 2026-02-22T22:12:59.770 INFO:teuthology.orchestra.run.trial026.stderr: 91.7% -- replaced with /var/log/ceph/cluster2-client.mirror.80988.log.gz 2026-02-22T22:12:59.770 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.65193.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.0.58951.log 2026-02-22T22:12:59.770 INFO:teuthology.orchestra.run.trial026.stderr: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.75851.log.gz 2026-02-22T22:12:59.770 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84071.log 2026-02-22T22:12:59.770 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.59140.log: /var/log/ceph/cluster2-client.mirror.0.58951.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48978.log 2026-02-22T22:12:59.771 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.84071.log: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.65193.log.gz 2026-02-22T22:12:59.771 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77069.log 2026-02-22T22:12:59.771 INFO:teuthology.orchestra.run.trial026.stderr: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.59140.log.gz 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.84071.log.gz 2026-02-22T22:12:59.771 INFO:teuthology.orchestra.run.trial026.stderr: 2026-02-22T22:12:59.771 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67084.log 2026-02-22T22:12:59.772 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.48978.log: /var/log/ceph/cluster2-client.mirror.77069.log: 57.7% -- replaced with /var/log/ceph/cluster2-client.mirror.48978.log.gz 2026-02-22T22:12:59.772 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79015.log 2026-02-22T22:12:59.772 INFO:teuthology.orchestra.run.trial026.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.77069.log.gz 2026-02-22T22:12:59.772 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.67084.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66146.log 2026-02-22T22:12:59.772 INFO:teuthology.orchestra.run.trial026.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.67084.log.gz 2026-02-22T22:12:59.772 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.79015.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.64790.log 2026-02-22T22:12:59.772 INFO:teuthology.orchestra.run.trial026.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.79015.log.gz 2026-02-22T22:12:59.773 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.66146.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61788.log 2026-02-22T22:12:59.774 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.64790.log: gzip 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.66146.log.gz 2026-02-22T22:12:59.774 INFO:teuthology.orchestra.run.trial026.stderr: -5 --verbose -- /var/log/ceph/cluster1-client.mirror.63846.log 2026-02-22T22:12:59.774 INFO:teuthology.orchestra.run.trial026.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.64790.log.gz 2026-02-22T22:12:59.774 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.61788.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71428.log 2026-02-22T22:12:59.774 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.63846.log: gzip -5 --verbose -- 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.61788.log.gz 2026-02-22T22:12:59.774 INFO:teuthology.orchestra.run.trial026.stderr: /var/log/ceph/cluster2-client.mirror.77420.log 2026-02-22T22:12:59.774 INFO:teuthology.orchestra.run.trial026.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.63846.log.gz 2026-02-22T22:12:59.774 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.71428.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54784.log 2026-02-22T22:12:59.775 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.77420.log: gzip -5 --verbose -- 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.71428.log.gz 2026-02-22T22:12:59.775 INFO:teuthology.orchestra.run.trial026.stderr: /var/log/ceph/cluster2-client.admin.47061.log 2026-02-22T22:12:59.775 INFO:teuthology.orchestra.run.trial026.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.77420.log.gz 2026-02-22T22:12:59.775 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.54784.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51815.log 2026-02-22T22:12:59.776 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.admin.47061.log: gzip 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.47061.log.gz 2026-02-22T22:12:59.776 INFO:teuthology.orchestra.run.trial026.stderr: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76511.log 2026-02-22T22:12:59.776 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.51815.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60858.log 2026-02-22T22:12:59.776 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.76511.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75500.log 2026-02-22T22:12:59.776 INFO:teuthology.orchestra.run.trial026.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.76511.log.gz 2026-02-22T22:12:59.777 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.60858.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76204.log 2026-02-22T22:12:59.777 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5/var/log/ceph/cluster2-client.mirror.75500.log: --verbose -- /var/log/ceph/cluster2-client.mirror.57412.log 2026-02-22T22:12:59.777 INFO:teuthology.orchestra.run.trial026.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.75500.log.gz 2026-02-22T22:12:59.778 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.76204.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.76204.log.gz 2026-02-22T22:12:59.778 INFO:teuthology.orchestra.run.trial026.stderr: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.54784.log.gz 2026-02-22T22:12:59.780 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72865.log 2026-02-22T22:12:59.780 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.57412.log: 93.6% -- replaced with /var/log/ceph/cluster2-client.mirror.51815.log.gz 2026-02-22T22:12:59.780 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51159.log 2026-02-22T22:12:59.780 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.72865.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58700.log 2026-02-22T22:12:59.780 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.51159.log: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.72865.log.gz 2026-02-22T22:12:59.780 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.3.62965.log 2026-02-22T22:12:59.780 INFO:teuthology.orchestra.run.trial026.stderr: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.51159.log.gz 2026-02-22T22:12:59.781 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.58700.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68471.log 2026-02-22T22:12:59.781 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.3.62965.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73712.log 2026-02-22T22:12:59.781 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.68471.log: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.58700.log.gz 2026-02-22T22:12:59.782 INFO:teuthology.orchestra.run.trial026.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.68471.log.gz 2026-02-22T22:12:59.782 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79285.log 2026-02-22T22:12:59.782 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.84878.log 2026-02-22T22:12:59.782 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.73712.log: /var/log/ceph/cluster1-client.mirror.79285.log: 86.5% -- replaced with /var/log/ceph/cluster1-client.mirror.79285.log.gz 2026-02-22T22:12:59.783 INFO:teuthology.orchestra.run.trial026.stderr: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.73712.log.gz 2026-02-22T22:12:59.789 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50730.log 2026-02-22T22:12:59.790 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68333.log 2026-02-22T22:12:59.790 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.admin.84878.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.84878.log.gz 2026-02-22T22:12:59.790 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.50730.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.83335.log 2026-02-22T22:12:59.791 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.68333.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.50730.log.gz 2026-02-22T22:12:59.791 INFO:teuthology.orchestra.run.trial026.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.68333.log.gzgzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59657.log 2026-02-22T22:12:59.791 INFO:teuthology.orchestra.run.trial026.stderr: 2026-02-22T22:12:59.792 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.83335.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.83335.log.gz 2026-02-22T22:12:59.792 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81159.log 2026-02-22T22:12:59.792 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69505.log 2026-02-22T22:12:59.792 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.59657.log: /var/log/ceph/cluster2-client.mirror.81159.log: 79.6% -- replaced with /var/log/ceph/cluster2-client.mirror.59657.log.gz 2026-02-22T22:12:59.793 INFO:teuthology.orchestra.run.trial026.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.81159.log.gz 2026-02-22T22:12:59.793 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.47400.log 2026-02-22T22:12:59.793 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53727.log 2026-02-22T22:12:59.793 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.69505.log: /var/log/ceph/cluster2-client.admin.47400.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.47400.log.gz 2026-02-22T22:12:59.794 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65656.log 2026-02-22T22:12:59.794 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.53727.log: 92.7% -- replaced with /var/log/ceph/cluster2-client.mirror.69505.log.gz 2026-02-22T22:12:59.795 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.48789.log 2026-02-22T22:12:59.796 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.65656.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.65656.log.gz 2026-02-22T22:12:59.796 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.74503.log 2026-02-22T22:12:59.797 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.48789.log: 54.5% -- replaced with /var/log/ceph/cluster1-client.mirror.48789.log.gz 2026-02-22T22:12:59.797 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55440.log 2026-02-22T22:12:59.798 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.74503.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.74503.log.gz 2026-02-22T22:12:59.798 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75583.log 2026-02-22T22:12:59.802 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.55440.log: 93.6% -- replaced with /var/log/ceph/cluster2-client.mirror.55440.log.gz 2026-02-22T22:12:59.802 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64015.log 2026-02-22T22:12:59.803 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.75583.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.75583.log.gz 2026-02-22T22:12:59.803 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62761.log 2026-02-22T22:12:59.803 INFO:teuthology.orchestra.run.trial026.stderr: 94.0% -- replaced with /var/log/ceph/cluster2-client.mirror.0.58951.log.gz 2026-02-22T22:12:59.804 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.64015.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.64015.log.gz 2026-02-22T22:12:59.804 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.62580.log 2026-02-22T22:12:59.805 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.62761.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.62217.log 2026-02-22T22:12:59.805 INFO:teuthology.orchestra.run.trial026.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.62761.log.gz 2026-02-22T22:12:59.805 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.62580.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65822.log 2026-02-22T22:12:59.806 INFO:teuthology.orchestra.run.trial026.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.62580.log.gz 2026-02-22T22:12:59.806 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.62217.log: 1.3% -- replaced with /var/log/ceph/cluster1-client.mirror.62217.log.gz 2026-02-22T22:12:59.806 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74983.log 2026-02-22T22:12:59.806 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65726.log 2026-02-22T22:12:59.807 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.65822.log: /var/log/ceph/cluster2-client.mirror.74983.log: 88.0% -- replaced with /var/log/ceph/cluster1-client.mirror.65822.log.gz 2026-02-22T22:12:59.807 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59774.log 2026-02-22T22:12:59.808 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.65726.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.65726.log.gz 2026-02-22T22:12:59.808 INFO:teuthology.orchestra.run.trial026.stderr: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.74983.log.gz 2026-02-22T22:12:59.808 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.73281.log 2026-02-22T22:12:59.808 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83478.log 2026-02-22T22:12:59.809 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.59774.log: /var/log/ceph/cluster1-client.mirror.73281.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.59774.log.gz 2026-02-22T22:12:59.809 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65497.log 2026-02-22T22:12:59.810 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.83478.log: 92.5% -- replaced with /var/log/ceph/cluster1-client.mirror.73281.log.gz 2026-02-22T22:12:59.810 INFO:teuthology.orchestra.run.trial026.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.83478.log.gz 2026-02-22T22:12:59.813 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55234.log 2026-02-22T22:12:59.814 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.65497.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79870.log 2026-02-22T22:12:59.814 INFO:teuthology.orchestra.run.trial026.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.65497.log.gz 2026-02-22T22:12:59.814 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.55234.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58613.log 2026-02-22T22:12:59.815 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.79870.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.79870.log.gz 2026-02-22T22:12:59.815 INFO:teuthology.orchestra.run.trial026.stderr: 90.1% -- replaced with /var/log/ceph/cluster2-client.mirror.55234.log.gz 2026-02-22T22:12:59.815 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52822.log 2026-02-22T22:12:59.815 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68198.log 2026-02-22T22:12:59.816 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.58613.log: /var/log/ceph/cluster2-client.mirror.52822.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.58613.log.gz 2026-02-22T22:12:59.817 INFO:teuthology.orchestra.run.trial026.stderr: 91.8% -- replaced with /var/log/ceph/cluster2-client.mirror.52822.log.gz 2026-02-22T22:12:59.817 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82537.log 2026-02-22T22:12:59.817 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81646.log 2026-02-22T22:12:59.817 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.68198.log: /var/log/ceph/cluster2-client.mirror.82537.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.68198.log.gz 2026-02-22T22:12:59.818 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81731.log 2026-02-22T22:12:59.818 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.81646.log: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.81646.log.gz 2026-02-22T22:12:59.819 INFO:teuthology.orchestra.run.trial026.stderr: 91.6%gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.47596.log 2026-02-22T22:12:59.819 INFO:teuthology.orchestra.run.trial026.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.82537.log.gz 2026-02-22T22:12:59.819 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.81731.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48009.log 2026-02-22T22:12:59.819 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.admin.47596.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.47596.log.gz 2026-02-22T22:12:59.820 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79083.log 2026-02-22T22:12:59.821 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.48009.log: 91.6% -- replaced with /var/log/ceph/cluster2-client.mirror.81731.log.gz 2026-02-22T22:12:59.822 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59587.log 2026-02-22T22:12:59.822 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.79083.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.79083.log.gz 2026-02-22T22:12:59.823 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50547.log 2026-02-22T22:12:59.824 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.59587.log: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.59587.log.gz 2026-02-22T22:12:59.824 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52056.log 2026-02-22T22:12:59.824 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.50547.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.50547.log.gz 2026-02-22T22:12:59.825 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81181.log 2026-02-22T22:12:59.825 INFO:teuthology.orchestra.run.trial026.stderr: 94.0% -- replaced with /var/log/ceph/cluster2-client.mirror.3.62965.log.gz/var/log/ceph/cluster2-client.mirror.52056.log: 2026-02-22T22:12:59.826 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51672.log 2026-02-22T22:12:59.826 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.81181.log: 90.0% -- replaced with /var/log/ceph/cluster2-client.mirror.52056.log.gz 2026-02-22T22:12:59.826 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66714.log 2026-02-22T22:12:59.827 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.51672.log: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.51672.log.gz 2026-02-22T22:12:59.828 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.74811.log 2026-02-22T22:12:59.829 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.66714.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.66714.log.gz 2026-02-22T22:12:59.829 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.62196.log 2026-02-22T22:12:59.830 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.74811.log: 92.8% -- replaced with /var/log/ceph/cluster1-client.mirror.74811.log.gz 2026-02-22T22:12:59.830 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78228.log 2026-02-22T22:12:59.831 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.62196.log: 1.3% -- replaced with /var/log/ceph/cluster1-client.mirror.62196.log.gz 2026-02-22T22:12:59.831 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.60370.log 2026-02-22T22:12:59.832 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.78228.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.78228.log.gz 2026-02-22T22:12:59.832 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68825.log 2026-02-22T22:12:59.845 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.3.60370.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70106.log 2026-02-22T22:12:59.845 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.68825.log: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.68825.log.gz 2026-02-22T22:12:59.858 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.62351.log 2026-02-22T22:12:59.859 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.70106.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.70106.log.gz 2026-02-22T22:12:59.867 INFO:teuthology.orchestra.run.trial026.stderr: 90.1% -- replaced with /var/log/ceph/cluster2-mon.a.log.gz 2026-02-22T22:12:59.867 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48150.log 2026-02-22T22:12:59.868 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.62351.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59487.log 2026-02-22T22:12:59.868 INFO:teuthology.orchestra.run.trial026.stderr: 87.4%/var/log/ceph/cluster2-client.mirror.48150.log: 78.2% -- replaced with /var/log/ceph/cluster2-client.mirror.48150.log.gz 2026-02-22T22:12:59.868 INFO:teuthology.orchestra.run.trial026.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.62351.log.gz 2026-02-22T22:12:59.869 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60551.log 2026-02-22T22:12:59.869 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.59487.log: 79.5% -- replaced with /var/log/ceph/cluster2-client.mirror.59487.log.gz 2026-02-22T22:12:59.870 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.64517.log 2026-02-22T22:12:59.870 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.60551.log: 79.7% -- replaced with /var/log/ceph/cluster2-client.mirror.60551.log.gz 2026-02-22T22:12:59.871 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78475.log 2026-02-22T22:12:59.871 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.64517.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.64517.log.gz 2026-02-22T22:12:59.872 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83187.log 2026-02-22T22:12:59.872 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.78475.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.78475.log.gz 2026-02-22T22:12:59.872 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69591.log 2026-02-22T22:12:59.873 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.83187.log: 79.6% -- replaced with /var/log/ceph/cluster2-client.mirror.83187.log.gz 2026-02-22T22:12:59.873 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60588.log 2026-02-22T22:12:59.874 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.69591.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.69591.log.gz 2026-02-22T22:12:59.875 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81811.log 2026-02-22T22:12:59.875 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.60588.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.60588.log.gz 2026-02-22T22:12:59.876 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84477.log 2026-02-22T22:12:59.880 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.81811.log: 94.4% -- replaced with /var/log/ceph/cluster2-client.mirror.48009.log.gz 2026-02-22T22:12:59.880 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.74174.log 2026-02-22T22:12:59.881 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.84477.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.84477.log.gz 2026-02-22T22:12:59.881 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56284.log 2026-02-22T22:12:59.882 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.74174.log: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.74174.log.gz 2026-02-22T22:12:59.882 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51794.log 2026-02-22T22:12:59.885 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.56284.log: 93.6% -- replaced with /var/log/ceph/cluster2-client.mirror.56284.log.gz 2026-02-22T22:12:59.886 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75541.log 2026-02-22T22:12:59.886 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.51794.log: 88.0% -- replaced with /var/log/ceph/cluster1-client.mirror.51794.log.gz 2026-02-22T22:12:59.887 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75665.log 2026-02-22T22:12:59.888 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.75541.log: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.75541.log.gz 2026-02-22T22:12:59.889 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74037.log 2026-02-22T22:12:59.890 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.75665.log: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.75665.log.gz 2026-02-22T22:12:59.891 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59570.log 2026-02-22T22:12:59.891 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.74037.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.74037.log.gz 2026-02-22T22:12:59.891 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.83521.log 2026-02-22T22:12:59.892 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.59570.log: 79.7% -- replaced with /var/log/ceph/cluster2-client.mirror.59570.log.gz 2026-02-22T22:12:59.892 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.1.48250.log 2026-02-22T22:12:59.893 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.83521.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.83521.log.gz 2026-02-22T22:12:59.893 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73595.log 2026-02-22T22:12:59.902 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.1.48250.log: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.1.48250.log.gz 2026-02-22T22:12:59.902 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.85374.log 2026-02-22T22:12:59.903 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.73595.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.73595.log.gz 2026-02-22T22:12:59.903 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.43793.log 2026-02-22T22:12:59.904 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.admin.85374.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.85374.log.gz 2026-02-22T22:12:59.905 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63732.log 2026-02-22T22:12:59.905 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.admin.43793.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.43793.log.gz 2026-02-22T22:12:59.905 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66244.log 2026-02-22T22:12:59.905 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.63732.log: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.63732.log.gz 2026-02-22T22:12:59.906 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77668.log 2026-02-22T22:12:59.907 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.66244.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.66244.log.gz 2026-02-22T22:12:59.907 INFO:teuthology.orchestra.run.trial026.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.61502.log.gz 2026-02-22T22:12:59.907 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72481.log 2026-02-22T22:12:59.908 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.77668.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.83270.log 2026-02-22T22:12:59.908 INFO:teuthology.orchestra.run.trial026.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.77668.log.gz 2026-02-22T22:12:59.908 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.72481.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.72481.log.gz 2026-02-22T22:12:59.911 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59737.log 2026-02-22T22:12:59.911 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.83270.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57214.log 2026-02-22T22:12:59.911 INFO:teuthology.orchestra.run.trial026.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.83270.log.gz 2026-02-22T22:12:59.911 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.59737.log: 79.6% -- replaced with /var/log/ceph/cluster2-client.mirror.59737.log.gz 2026-02-22T22:12:59.911 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62025.log 2026-02-22T22:12:59.911 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.57214.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58512.log 2026-02-22T22:12:59.911 INFO:teuthology.orchestra.run.trial026.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.57214.log.gz 2026-02-22T22:12:59.911 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.62025.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79690.log 2026-02-22T22:12:59.911 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.58512.log: 79.6% -- replaced with /var/log/ceph/cluster2-client.mirror.58512.log.gz 2026-02-22T22:12:59.911 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84368.log 2026-02-22T22:12:59.911 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.79690.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.79690.log.gz 2026-02-22T22:12:59.913 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.82728.log 2026-02-22T22:12:59.913 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.84368.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.84368.log.gz 2026-02-22T22:12:59.913 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84660.log 2026-02-22T22:12:59.913 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.82728.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.82728.log.gz 2026-02-22T22:12:59.914 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.64589.log 2026-02-22T22:12:59.914 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.84660.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.mirror.84660.log.gz 2026-02-22T22:12:59.914 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78543.log 2026-02-22T22:12:59.914 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.64589.log: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.64589.log.gz 2026-02-22T22:12:59.915 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77503.log 2026-02-22T22:12:59.915 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.78543.log: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.78543.log.gz 2026-02-22T22:12:59.916 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77895.log 2026-02-22T22:12:59.916 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.77503.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.77503.log.gz 2026-02-22T22:12:59.917 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.64087.log 2026-02-22T22:12:59.918 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.77895.log: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.77895.log.gz 2026-02-22T22:12:59.919 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.1.57004.log 2026-02-22T22:12:59.919 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.64087.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.64087.log.gz 2026-02-22T22:12:59.920 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.46812.log 2026-02-22T22:12:59.929 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.1.57004.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.48009.log 2026-02-22T22:12:59.935 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.admin.46812.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.46812.log.gz 2026-02-22T22:12:59.938 INFO:teuthology.orchestra.run.trial026.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.1.57004.log.gz 2026-02-22T22:12:59.938 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.64377.log 2026-02-22T22:12:59.944 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.0.48009.log: 93.2% -- replaced with /var/log/ceph/cluster1-client.mirror.3.60370.log.gz 2026-02-22T22:12:59.945 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.71277.log 2026-02-22T22:12:59.946 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.64377.log: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.64377.log.gz 2026-02-22T22:12:59.946 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60284.log 2026-02-22T22:12:59.947 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.71277.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.71277.log.gz 2026-02-22T22:12:59.947 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53110.log 2026-02-22T22:12:59.948 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.60284.log: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.60284.log.gz 2026-02-22T22:12:59.949 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65173.log 2026-02-22T22:12:59.953 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.53110.log: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.53110.log.gz 2026-02-22T22:12:59.953 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71567.log 2026-02-22T22:12:59.954 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.65173.log: 92.7% -- replaced with /var/log/ceph/cluster2-client.mirror.65173.log.gz 2026-02-22T22:12:59.955 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66735.log 2026-02-22T22:12:59.955 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.71567.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.71567.log.gz 2026-02-22T22:12:59.956 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53705.log 2026-02-22T22:12:59.957 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.66735.log: 92.7% -- replaced with /var/log/ceph/cluster2-client.mirror.66735.log.gz 2026-02-22T22:12:59.958 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.63621.log 2026-02-22T22:12:59.958 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.53705.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.53705.log.gz 2026-02-22T22:12:59.959 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55156.log 2026-02-22T22:12:59.959 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.63621.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.63621.log.gz 2026-02-22T22:12:59.960 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75004.log 2026-02-22T22:12:59.964 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.55156.log: 93.6% -- replaced with /var/log/ceph/cluster2-client.mirror.55156.log.gz 2026-02-22T22:12:59.964 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82496.log 2026-02-22T22:12:59.965 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.75004.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.75004.log.gz 2026-02-22T22:12:59.965 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52866.log 2026-02-22T22:12:59.966 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.82496.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.82496.log.gz 2026-02-22T22:12:59.966 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74663.log 2026-02-22T22:12:59.967 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.52866.log: 77.5% -- replaced with /var/log/ceph/cluster2-client.mirror.52866.log.gz 2026-02-22T22:12:59.967 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.50436.log 2026-02-22T22:12:59.968 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.74663.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.74663.log.gz 2026-02-22T22:12:59.968 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84562.log 2026-02-22T22:12:59.969 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.admin.50436.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.50436.log.gz 2026-02-22T22:12:59.969 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82575.log 2026-02-22T22:12:59.970 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.84562.log: 77.0% -- replaced with /var/log/ceph/cluster2-client.mirror.84562.log.gz 2026-02-22T22:12:59.970 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66363.log 2026-02-22T22:12:59.971 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.82575.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.82575.log.gz 2026-02-22T22:12:59.971 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67775.log 2026-02-22T22:12:59.972 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.66363.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.66363.log.gz 2026-02-22T22:12:59.973 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79533.log 2026-02-22T22:12:59.973 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.67775.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.67775.log.gz 2026-02-22T22:12:59.974 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74767.log 2026-02-22T22:12:59.974 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.79533.log: 86.6% -- replaced with /var/log/ceph/cluster1-client.mirror.79533.log.gz 2026-02-22T22:12:59.975 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56567.log 2026-02-22T22:12:59.975 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.74767.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.74767.log.gz 2026-02-22T22:12:59.976 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75748.log 2026-02-22T22:12:59.976 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.56567.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.56567.log.gz 2026-02-22T22:12:59.977 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77131.log 2026-02-22T22:12:59.978 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.75748.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.75748.log.gz 2026-02-22T22:12:59.978 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.48721.log 2026-02-22T22:12:59.979 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.77131.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.77131.log.gz 2026-02-22T22:12:59.979 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.63645.log 2026-02-22T22:12:59.980 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.48721.log: 56.7% -- replaced with /var/log/ceph/cluster1-client.mirror.48721.log.gz 2026-02-22T22:12:59.980 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59040.log 2026-02-22T22:12:59.981 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.63645.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.63645.log.gz 2026-02-22T22:12:59.981 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71762.log 2026-02-22T22:12:59.981 INFO:teuthology.orchestra.run.trial026.stderr: 94.6% -- replaced with /var/log/ceph/cluster1-client.mirror.0.48009.log.gz 2026-02-22T22:12:59.982 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.59040.log: 79.7%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56952.log 2026-02-22T22:12:59.982 INFO:teuthology.orchestra.run.trial026.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.59040.log.gz 2026-02-22T22:12:59.982 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.71762.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58826.log 2026-02-22T22:12:59.982 INFO:teuthology.orchestra.run.trial026.stderr: 89.7%/var/log/ceph/cluster2-client.mirror.56952.log: -- replaced with /var/log/ceph/cluster2-client.mirror.71762.log.gz 2026-02-22T22:12:59.983 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.80073.log 2026-02-22T22:12:59.993 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.58826.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70711.log 2026-02-22T22:12:59.993 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.80073.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.80073.log.gz 2026-02-22T22:13:00.007 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66835.log 2026-02-22T22:13:00.008 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.70711.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.70711.log.gz 2026-02-22T22:13:00.020 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.54473.log 2026-02-22T22:13:00.021 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.66835.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.66835.log.gz 2026-02-22T22:13:00.024 INFO:teuthology.orchestra.run.trial026.stderr: 93.9% -- replaced with /var/log/ceph/cluster2-client.mirror.59629.log.gz 2026-02-22T22:13:00.025 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69776.log 2026-02-22T22:13:00.033 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.3.54473.log: 93.9% -- replaced with /var/log/ceph/cluster2-client.mirror.61585.log.gz 2026-02-22T22:13:00.033 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.72553.log 2026-02-22T22:13:00.034 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.69776.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.69776.log.gz 2026-02-22T22:13:00.035 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50521.log 2026-02-22T22:13:00.035 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.72553.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.72553.log.gz 2026-02-22T22:13:00.035 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55569.log 2026-02-22T22:13:00.037 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.50521.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.50521.log.gz 2026-02-22T22:13:00.038 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78925.log 2026-02-22T22:13:00.040 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.55569.log: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.55569.log.gz 2026-02-22T22:13:00.043 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.62604.log 2026-02-22T22:13:00.043 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.78925.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.78925.log.gz 2026-02-22T22:13:00.043 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68116.log 2026-02-22T22:13:00.043 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.62604.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.62604.log.gz 2026-02-22T22:13:00.043 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55741.log 2026-02-22T22:13:00.044 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.68116.log: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.68116.log.gz 2026-02-22T22:13:00.044 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.43746.log 2026-02-22T22:13:00.046 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.55741.log: 93.0% -- replaced with /var/log/ceph/cluster1-client.mirror.3.54473.log.gz 2026-02-22T22:13:00.047 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.71615.log 2026-02-22T22:13:00.048 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.admin.43746.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.43746.log.gz 2026-02-22T22:13:00.048 INFO:teuthology.orchestra.run.trial026.stderr: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.55741.log.gz 2026-02-22T22:13:00.048 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68222.log 2026-02-22T22:13:00.048 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.71615.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.71615.log.gz 2026-02-22T22:13:00.049 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51414.log 2026-02-22T22:13:00.049 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose/var/log/ceph/cluster1-client.mirror.68222.log: -- /var/log/ceph/cluster2-client.mirror.77461.log 2026-02-22T22:13:00.049 INFO:teuthology.orchestra.run.trial026.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.68222.log.gz 2026-02-22T22:13:00.050 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.51414.log: 91.7% -- replaced with /var/log/ceph/cluster2-client.mirror.51414.log.gz 2026-02-22T22:13:00.050 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79375.log 2026-02-22T22:13:00.051 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.77461.log: gzip -5 --verbose -- /var/log/ceph/cluster2-osd.2.log 2026-02-22T22:13:00.051 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.79375.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.79375.log.gz 2026-02-22T22:13:00.051 INFO:teuthology.orchestra.run.trial026.stderr: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.77461.log.gz 2026-02-22T22:13:00.052 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78340.log 2026-02-22T22:13:00.052 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-osd.2.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70805.log 2026-02-22T22:13:00.053 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.78340.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.78340.log.gz 2026-02-22T22:13:00.055 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53130.log 2026-02-22T22:13:00.056 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.70805.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.70805.log.gz 2026-02-22T22:13:00.056 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67582.log 2026-02-22T22:13:00.059 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.53130.log: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.53130.log.gz 2026-02-22T22:13:00.060 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59268.log 2026-02-22T22:13:00.060 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.67582.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.67582.log.gz 2026-02-22T22:13:00.061 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75521.log 2026-02-22T22:13:00.072 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.59268.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76717.log 2026-02-22T22:13:00.072 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.75521.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.75521.log.gz 2026-02-22T22:13:00.079 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52993.log 2026-02-22T22:13:00.092 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.76717.log: 90.6%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62003.log 2026-02-22T22:13:00.092 INFO:teuthology.orchestra.run.trial026.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.76717.log.gz 2026-02-22T22:13:00.092 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.52993.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.52993.log.gz 2026-02-22T22:13:00.099 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70669.log 2026-02-22T22:13:00.099 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.62003.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.62003.log.gz 2026-02-22T22:13:00.108 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75769.log 2026-02-22T22:13:00.121 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.70669.log: 92.6%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49046.log 2026-02-22T22:13:00.122 INFO:teuthology.orchestra.run.trial026.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.70669.log.gz 2026-02-22T22:13:00.122 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.75769.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.75769.log.gz 2026-02-22T22:13:00.124 INFO:teuthology.orchestra.run.trial026.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.82983.log.gz 2026-02-22T22:13:00.125 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70299.log 2026-02-22T22:13:00.125 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.49046.log: 54.5% -- replaced with /var/log/ceph/cluster2-client.mirror.49046.log.gz 2026-02-22T22:13:00.126 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77544.log 2026-02-22T22:13:00.126 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.70299.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.70299.log.gz 2026-02-22T22:13:00.126 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59123.log 2026-02-22T22:13:00.127 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.77544.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.77544.log.gz 2026-02-22T22:13:00.127 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57926.log 2026-02-22T22:13:00.129 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.59123.log: 79.5% -- replaced with /var/log/ceph/cluster2-client.mirror.59123.log.gz 2026-02-22T22:13:00.129 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83441.log 2026-02-22T22:13:00.131 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.57926.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.57926.log.gz 2026-02-22T22:13:00.132 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65521.log 2026-02-22T22:13:00.132 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.83441.log: 79.6% -- replaced with /var/log/ceph/cluster2-client.mirror.83441.log.gz 2026-02-22T22:13:00.132 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58951.log 2026-02-22T22:13:00.132 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.65521.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.65521.log.gz 2026-02-22T22:13:00.132 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58720.log 2026-02-22T22:13:00.133 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.58951.log: 95.0% -- replaced with /var/log/ceph/cluster1-client.mirror.58951.log.gz 2026-02-22T22:13:00.135 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58549.log 2026-02-22T22:13:00.135 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.58720.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.58720.log.gz 2026-02-22T22:13:00.135 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84224.log 2026-02-22T22:13:00.136 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.58549.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.58549.log.gz 2026-02-22T22:13:00.136 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78813.log 2026-02-22T22:13:00.136 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster2-client.mirror.84224.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.84224.log.gz 2026-02-22T22:13:00.137 INFO:teuthology.orchestra.run.trial026.stderr:/var/log/ceph/cluster1-client.mirror.78813.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.78813.log.gz 2026-02-22T22:13:00.142 INFO:teuthology.orchestra.run.trial026.stderr: 93.9% -- replaced with /var/log/ceph/cluster2-client.mirror.58406.log.gz 2026-02-22T22:13:00.194 INFO:teuthology.orchestra.run.trial026.stderr: 93.9% -- replaced with /var/log/ceph/cluster2-client.mirror.57323.log.gz 2026-02-22T22:13:00.205 INFO:teuthology.orchestra.run.trial026.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.60610.log.gz 2026-02-22T22:13:00.247 INFO:teuthology.orchestra.run.trial026.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.60858.log.gz 2026-02-22T22:13:00.247 INFO:teuthology.orchestra.run.trial026.stderr: 93.9% -- replaced with /var/log/ceph/cluster2-client.mirror.81811.log.gz 2026-02-22T22:13:00.278 INFO:teuthology.orchestra.run.trial026.stderr: 93.9% -- replaced with /var/log/ceph/cluster2-client.mirror.53727.log.gz 2026-02-22T22:13:00.297 INFO:teuthology.orchestra.run.trial026.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.60939.log.gz 2026-02-22T22:13:00.324 INFO:teuthology.orchestra.run.trial026.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.57412.log.gz 2026-02-22T22:13:00.333 INFO:teuthology.orchestra.run.trial026.stderr: 93.9% -- replaced with /var/log/ceph/cluster2-client.mirror.50752.log.gz 2026-02-22T22:13:00.335 INFO:teuthology.orchestra.run.trial026.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.82344.log.gz 2026-02-22T22:13:00.351 INFO:teuthology.orchestra.run.trial026.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.59796.log.gz 2026-02-22T22:13:00.381 INFO:teuthology.orchestra.run.trial026.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.58032.log.gz 2026-02-22T22:13:00.431 INFO:teuthology.orchestra.run.trial026.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.62025.log.gz 2026-02-22T22:13:00.485 INFO:teuthology.orchestra.run.trial026.stderr: 93.9% -- replaced with /var/log/ceph/cluster2-client.mirror.81181.log.gz 2026-02-22T22:13:00.526 INFO:teuthology.orchestra.run.trial026.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.52330.log.gz 2026-02-22T22:13:00.536 INFO:teuthology.orchestra.run.trial026.stderr: 93.9% -- replaced with /var/log/ceph/cluster2-client.mirror.54828.log.gz 2026-02-22T22:13:00.546 INFO:teuthology.orchestra.run.trial026.stderr: 93.9% -- replaced with /var/log/ceph/cluster2-client.mirror.83246.log.gz 2026-02-22T22:13:00.595 INFO:teuthology.orchestra.run.trial026.stderr: 93.9% -- replaced with /var/log/ceph/cluster2-client.mirror.58826.log.gz 2026-02-22T22:13:00.706 INFO:teuthology.orchestra.run.trial026.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.59268.log.gz 2026-02-22T22:13:00.729 INFO:teuthology.orchestra.run.trial026.stderr: 93.9% -- replaced with /var/log/ceph/cluster2-client.mirror.56952.log.gz 2026-02-22T22:13:02.664 INFO:teuthology.orchestra.run.trial004.stderr: 94.4% -- replaced with /var/log/ceph/cluster1-osd.1.log.gz 2026-02-22T22:13:02.875 INFO:teuthology.orchestra.run.trial004.stderr: 94.3% -- replaced with /var/log/ceph/cluster1-osd.0.log.gz 2026-02-22T22:13:02.919 INFO:teuthology.orchestra.run.trial004.stderr: 94.5% -- replaced with /var/log/ceph/cluster1-osd.2.log.gz 2026-02-22T22:13:02.920 INFO:teuthology.orchestra.run.trial004.stderr: 2026-02-22T22:13:02.920 INFO:teuthology.orchestra.run.trial004.stderr:real 0m4.958s 2026-02-22T22:13:02.920 INFO:teuthology.orchestra.run.trial004.stderr:user 0m14.586s 2026-02-22T22:13:02.921 INFO:teuthology.orchestra.run.trial004.stderr:sys 0m0.731s 2026-02-22T22:14:25.237 INFO:teuthology.orchestra.run.trial026.stderr: 93.4% -- replaced with /var/log/ceph/cluster2-osd.1.log.gz 2026-02-22T22:14:31.264 INFO:teuthology.orchestra.run.trial026.stderr: 93.4% -- replaced with /var/log/ceph/cluster2-osd.2.log.gz 2026-02-22T22:15:00.379 INFO:teuthology.orchestra.run.trial026.stderr: 93.4% -- replaced with /var/log/ceph/cluster2-osd.0.log.gz 2026-02-22T22:15:00.381 INFO:teuthology.orchestra.run.trial026.stderr: 2026-02-22T22:15:00.381 INFO:teuthology.orchestra.run.trial026.stderr:real 2m2.396s 2026-02-22T22:15:00.381 INFO:teuthology.orchestra.run.trial026.stderr:user 5m34.351s 2026-02-22T22:15:00.381 INFO:teuthology.orchestra.run.trial026.stderr:sys 0m16.337s 2026-02-22T22:15:00.381 INFO:tasks.ceph:Archiving logs... 2026-02-22T22:15:00.382 DEBUG:teuthology.misc:Transferring archived files from trial004:/var/log/ceph to /home/teuthworker/mnt/teuthology/yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial/64060/remote/trial004/log 2026-02-22T22:15:00.383 DEBUG:teuthology.orchestra.run.trial004:> sudo tar c -f - -C /var/log/ceph -- . 2026-02-22T22:15:01.532 DEBUG:teuthology.misc:Transferring archived files from trial026:/var/log/ceph to /home/teuthworker/mnt/teuthology/yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial/64060/remote/trial026/log 2026-02-22T22:15:01.533 DEBUG:teuthology.orchestra.run.trial026:> sudo tar c -f - -C /var/log/ceph -- . 2026-02-22T22:15:26.090 DEBUG:teuthology.run_tasks:Unwinding manager install 2026-02-22T22:15:26.102 INFO:teuthology.task.install.util:Removing shipped files: /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer... 2026-02-22T22:15:26.102 DEBUG:teuthology.orchestra.run.trial004:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2026-02-22T22:15:26.104 DEBUG:teuthology.orchestra.run.trial026:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2026-02-22T22:15:26.255 DEBUG:teuthology.run_tasks:Unwinding manager clock 2026-02-22T22:15:26.267 INFO:teuthology.task.clock:Checking final clock skew... 2026-02-22T22:15:26.267 DEBUG:teuthology.orchestra.run.trial004:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2026-02-22T22:15:26.269 DEBUG:teuthology.orchestra.run.trial026:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2026-02-22T22:15:26.283 INFO:teuthology.orchestra.run.trial004.stderr:bash: line 1: ntpq: command not found 2026-02-22T22:15:26.285 INFO:teuthology.orchestra.run.trial026.stderr:bash: line 1: ntpq: command not found 2026-02-22T22:15:31.314 INFO:teuthology.orchestra.run.trial004.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2026-02-22T22:15:31.314 INFO:teuthology.orchestra.run.trial004.stdout:=============================================================================== 2026-02-22T22:15:31.314 INFO:teuthology.orchestra.run.trial004.stdout:^- 104-167-215-195.ipv4.ber> 2 8 353 189 +4157us[+4142us] +/- 35ms 2026-02-22T22:15:31.314 INFO:teuthology.orchestra.run.trial004.stdout:^- 23.186.168.129 2 8 367 324 +620us[ +606us] +/- 55ms 2026-02-22T22:15:31.314 INFO:teuthology.orchestra.run.trial004.stdout:^- dns-e.ns4v.icu 2 8 377 132 +2056us[+2040us] +/- 73ms 2026-02-22T22:15:31.314 INFO:teuthology.orchestra.run.trial004.stdout:^* nyc3.us.ntp.li 2 8 377 69 -206us[ -222us] +/- 2553us 2026-02-22T22:15:31.315 INFO:teuthology.orchestra.run.trial026.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2026-02-22T22:15:31.315 INFO:teuthology.orchestra.run.trial026.stdout:=============================================================================== 2026-02-22T22:15:31.315 INFO:teuthology.orchestra.run.trial026.stdout:^- 104-167-215-195.ipv4.ber> 2 8 365 125 +3263us[+3155us] +/- 36ms 2026-02-22T22:15:31.315 INFO:teuthology.orchestra.run.trial026.stdout:^- 23.186.168.129 2 7 177 254 +660us[ +519us] +/- 56ms 2026-02-22T22:15:31.315 INFO:teuthology.orchestra.run.trial026.stdout:^- dns-e.ns4v.icu 2 7 377 5 +2053us[+2053us] +/- 75ms 2026-02-22T22:15:31.315 INFO:teuthology.orchestra.run.trial026.stdout:^* nyc3.us.ntp.li 2 6 377 10 -57us[ -178us] +/- 2394us 2026-02-22T22:15:31.315 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2026-02-22T22:15:31.328 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2026-02-22T22:15:31.328 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2026-02-22T22:15:31.343 DEBUG:teuthology.orchestra.run.trial004:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2026-02-22T22:15:31.410 DEBUG:teuthology.orchestra.run.trial026:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2026-02-22T22:15:31.469 DEBUG:teuthology.orchestra.run.trial004:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:getty_t:s0 2026-02-22T22:15:31.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-22T22:15:31.498 DEBUG:teuthology.orchestra.run.trial026:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:getty_t:s0 2026-02-22T22:15:31.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-22T22:15:31.525 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2026-02-22T22:15:31.546 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2026-02-22T22:15:31.567 INFO:teuthology.task.internal:Duration was 2630.964098 seconds 2026-02-22T22:15:31.567 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2026-02-22T22:15:31.587 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2026-02-22T22:15:31.588 DEBUG:teuthology.orchestra.run.trial004:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2026-02-22T22:15:31.590 DEBUG:teuthology.orchestra.run.trial026:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2026-02-22T22:15:31.625 INFO:teuthology.orchestra.run.trial004.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2026-02-22T22:15:31.627 INFO:teuthology.orchestra.run.trial026.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2026-02-22T22:15:31.843 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2026-02-22T22:15:31.844 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@trial004.front.sepia.ceph.com 2026-02-22T22:15:31.844 DEBUG:teuthology.orchestra.run.trial004:> grep -E --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | grep -E -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | grep -E -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2026-02-22T22:15:31.865 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@trial026.front.sepia.ceph.com 2026-02-22T22:15:31.865 DEBUG:teuthology.orchestra.run.trial026:> grep -E --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | grep -E -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | grep -E -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2026-02-22T22:15:31.904 INFO:teuthology.task.internal.syslog:Gathering journactl... 2026-02-22T22:15:31.904 DEBUG:teuthology.orchestra.run.trial004:> sudo journalctl > /home/ubuntu/cephtest/archive/syslog/journalctl.log 2026-02-22T22:15:31.907 DEBUG:teuthology.orchestra.run.trial026:> sudo journalctl > /home/ubuntu/cephtest/archive/syslog/journalctl.log 2026-02-22T22:15:32.008 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2026-02-22T22:15:32.008 DEBUG:teuthology.orchestra.run.trial004:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --max-args=1 --max-procs=0 --verbose --no-run-if-empty -- gzip -5 --verbose -- 2026-02-22T22:15:32.011 DEBUG:teuthology.orchestra.run.trial026:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --max-args=1 --max-procs=0 --verbose --no-run-if-empty -- gzip -5 --verbose -- 2026-02-22T22:15:32.032 INFO:teuthology.orchestra.run.trial004.stderr:gzip -5 --verbose -- /home/ubuntu/cephtest/archive/syslog/misc.log 2026-02-22T22:15:32.034 INFO:teuthology.orchestra.run.trial004.stderr:gzip -5 --verbose -- /home/ubuntu/cephtest/archive/syslog/kern.log 2026-02-22T22:15:32.034 INFO:teuthology.orchestra.run.trial004.stderr:/home/ubuntu/cephtest/archive/syslog/misc.log: gzip -5 --verbose -- /home/ubuntu/cephtest/archive/syslog/journalctl.log 2026-02-22T22:15:32.034 INFO:teuthology.orchestra.run.trial004.stderr: 0.0% -- replaced with /home/ubuntu/cephtest/archive/syslog/misc.log.gz 2026-02-22T22:15:32.034 INFO:teuthology.orchestra.run.trial004.stderr:/home/ubuntu/cephtest/archive/syslog/kern.log: 0.0% -- replaced with /home/ubuntu/cephtest/archive/syslog/kern.log.gz 2026-02-22T22:15:32.035 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /home/ubuntu/cephtest/archive/syslog/misc.log 2026-02-22T22:15:32.035 INFO:teuthology.orchestra.run.trial026.stderr:gzip -5 --verbose -- /home/ubuntu/cephtest/archive/syslog/kern.log 2026-02-22T22:15:32.036 INFO:teuthology.orchestra.run.trial026.stderr:/home/ubuntu/cephtest/archive/syslog/misc.log: gzip -5 --verbose -- /home/ubuntu/cephtest/archive/syslog/journalctl.log 2026-02-22T22:15:32.036 INFO:teuthology.orchestra.run.trial026.stderr: 0.0% -- replaced with /home/ubuntu/cephtest/archive/syslog/misc.log.gz 2026-02-22T22:15:32.036 INFO:teuthology.orchestra.run.trial026.stderr:/home/ubuntu/cephtest/archive/syslog/kern.log: 0.0% -- replaced with /home/ubuntu/cephtest/archive/syslog/kern.log.gz 2026-02-22T22:15:32.043 INFO:teuthology.orchestra.run.trial004.stderr:/home/ubuntu/cephtest/archive/syslog/journalctl.log: 82.5% -- replaced with /home/ubuntu/cephtest/archive/syslog/journalctl.log.gz 2026-02-22T22:15:32.046 INFO:teuthology.orchestra.run.trial026.stderr:/home/ubuntu/cephtest/archive/syslog/journalctl.log: 82.6% -- replaced with /home/ubuntu/cephtest/archive/syslog/journalctl.log.gz 2026-02-22T22:15:32.047 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2026-02-22T22:15:32.069 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2026-02-22T22:15:32.070 DEBUG:teuthology.orchestra.run.trial004:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2026-02-22T22:15:32.112 DEBUG:teuthology.orchestra.run.trial026:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2026-02-22T22:15:32.136 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2026-02-22T22:15:32.158 DEBUG:teuthology.orchestra.run.trial004:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2026-02-22T22:15:32.160 DEBUG:teuthology.orchestra.run.trial026:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2026-02-22T22:15:32.182 INFO:teuthology.orchestra.run.trial004.stdout:kernel.core_pattern = core 2026-02-22T22:15:32.199 INFO:teuthology.orchestra.run.trial026.stdout:kernel.core_pattern = core 2026-02-22T22:15:32.214 DEBUG:teuthology.orchestra.run.trial004:> test -e /home/ubuntu/cephtest/archive/coredump 2026-02-22T22:15:32.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-22T22:15:32.258 DEBUG:teuthology.orchestra.run.trial026:> test -e /home/ubuntu/cephtest/archive/coredump 2026-02-22T22:15:32.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-22T22:15:32.273 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2026-02-22T22:15:32.286 INFO:teuthology.task.internal:Transferring archived files... 2026-02-22T22:15:32.286 DEBUG:teuthology.misc:Transferring archived files from trial004:/home/ubuntu/cephtest/archive to /home/teuthworker/mnt/teuthology/yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial/64060/remote/trial004 2026-02-22T22:15:32.286 DEBUG:teuthology.orchestra.run.trial004:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2026-02-22T22:15:32.328 DEBUG:teuthology.misc:Transferring archived files from trial026:/home/ubuntu/cephtest/archive to /home/teuthworker/mnt/teuthology/yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial/64060/remote/trial026 2026-02-22T22:15:32.328 DEBUG:teuthology.orchestra.run.trial026:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2026-02-22T22:15:32.358 INFO:teuthology.task.internal:Removing archive directory... 2026-02-22T22:15:32.358 DEBUG:teuthology.orchestra.run.trial004:> rm -rf -- /home/ubuntu/cephtest/archive 2026-02-22T22:15:32.363 DEBUG:teuthology.orchestra.run.trial026:> rm -rf -- /home/ubuntu/cephtest/archive 2026-02-22T22:15:32.411 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2026-02-22T22:15:32.424 INFO:teuthology.task.internal:Not uploading archives. 2026-02-22T22:15:32.424 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2026-02-22T22:15:32.454 INFO:teuthology.task.internal:Tidying up after the test... 2026-02-22T22:15:32.454 DEBUG:teuthology.orchestra.run.trial004:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2026-02-22T22:15:32.457 DEBUG:teuthology.orchestra.run.trial026:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2026-02-22T22:15:32.471 INFO:teuthology.orchestra.run.trial004.stdout: 917521 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Feb 22 22:15 /home/ubuntu/cephtest 2026-02-22T22:15:32.474 INFO:teuthology.orchestra.run.trial026.stdout: 917572 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Feb 22 22:15 /home/ubuntu/cephtest 2026-02-22T22:15:32.475 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2026-02-22T22:15:32.488 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2026-02-22T22:15:32.518 INFO:teuthology.run:Summary data: description: rbd/mirror-thrash/{base/install clients/mirror cluster/{2-node openstack} conf/{disable-pool-app} msgr-failures/few objectstore/bluestore-comp-lz4 policy/simple rbd-mirror/four-per-cluster supported-random-distro$/{centos_latest} workloads/rbd-mirror-snapshot-stress-workunit-exclusive-lock} duration: 2630.9640979766846 flavor: default owner: scheduled_yuriw@soko04.front.sepia.ceph.com success: true 2026-02-22T22:15:32.518 DEBUG:teuthology.report:Pushing job info to https://paddles-paddles.apps.pok.os.sepia.ceph.com 2026-02-22T22:15:32.630 INFO:teuthology.run:pass